Exemplo n.º 1
0
    def _parse_flags(self, cmd_flag_map, args):
        """Expects arguments after the initial command and subcommands (i.e.,
        the second item returned from _parse_subcmds)

        Returns a tuple of (multidict of flag names to parsed and validated values, remaining_args).

        Raises on unknown subcommands.
        """
        flag_value_map = OMD()
        ff_path_res_map = OrderedDict()
        ff_path_seen = set()

        orig_args = args
        while args:
            arg = args[0]
            if not arg or arg[0] != '-' or arg == '-' or arg == '--':
                # posargs or post_posargs beginning ('-' is a conventional pos arg for stdin)
                break
            flag, value, args = self._parse_single_flag(cmd_flag_map, args)
            flag_value_map.add(flag.name, value)

            if flag is self.flagfile_flag:
                self._parse_flagfile(cmd_flag_map,
                                     value,
                                     res_map=ff_path_res_map)
                for path, ff_flag_value_map in ff_path_res_map.items():
                    if path in ff_path_seen:
                        continue
                    flag_value_map.update_extend(ff_flag_value_map)
                    ff_path_seen.add(path)

        return flag_value_map, ff_path_res_map, args
Exemplo n.º 2
0
def find_instruction_order(instructions: OrderedMultiDict):
    starting_instruction = set(instructions.keys(multi=True)).difference(
        set(instructions.values(multi=True)))
    worker_manager = WorkerManager(offset=0)
    instruction_order = IndexedSet()
    current_instructions = list()
    for index, value in enumerate(sorted(starting_instruction)):
        current_instructions.append(value)
        worker_manager.add(value)
    current_instructions = sorted(current_instructions)
    while current_instructions:
        index = -1
        for i, current_instruction in enumerate(current_instructions):
            if not {
                    key
                    for key, value in instructions.items(multi=True)
                    if value == current_instruction
            }.difference(instruction_order):
                instruction_order.add(current_instruction)
                worker_manager.remove(current_instruction)
                for n, value in enumerate(
                        instructions.getlist(current_instruction)):
                    if value not in current_instructions:
                        current_instructions.append(value)
                index = i
                break
        if index != -1:
            current_instructions.pop(index)
        current_instructions = sorted(current_instructions)
        for instruction in current_instructions:
            worker_manager.add(instruction)
    print("".join(instruction_order))
    print(worker_manager.start_time)
Exemplo n.º 3
0
class EventEmitter2(object):

    def __init__(self):
        """
        """
        self.events = OrderedMultiDict()
        self.on = self.add_listener
        self.off = self.remove_listener
        # TODO: Add listener to remove

    def add_listener(self, event, listener, count=0):
        if not (isinstance(listener, FunctionType)) or (isinstance(listener, BuiltinFunctionType)):
            raise Exception("Invalid Listener: %s" % (str(listener)))
        _event = re.compile(event)
        _listener = {"handler": listener, "calls": 0, "calls_left": count}
        self.events.add(_event, _listener)
        return True

    def emit(self, event, kwargs):
        for pattern, listener in self.events.iteritems(multi=True):
            if pattern.match(event):
                if not listener["calls_left"]:
                    log.debug("Removing Listener: %s on Pattern: %s") % (
                        listener, pattern)
                    self.remove_listener(pattern, listener)
                listener["calls"] += 1
                listener["calls_left"] -= 1
                yield listener["handler"](**kwargs)

    def remove_listener(self, pattern, listener):
        pattern = re.compile(pattern)
        listeners = self.events.getlist(pattern)
        for pattern, _listener in self.events.iteritems(multi=True):
            if _listener['handler'] == listener:
                listener = _listener
                break
        listeners = self.events.getlist(pattern)
        listeners.remove(listener)
        if len(listeners):
            self.events.update({pattern: listeners})
        else:
            self.events._remove(pattern)
        return True

    def on_any(self, listener):
        raise NotImplementedError

    def off_any(self, listener):
        raise NotImplementedError

    def once(self, event, listener):
        self.add_listener(event, listener, left=1)

    def many(self, event, listener, left):
        self.add_listener(event, listener, left)
Exemplo n.º 4
0
Arquivo: core.py Projeto: ra2003/chert
 def validate(self):
     self._call_custom_hook('pre_validate')
     dup_id_map = {}
     eid_map = OMD([(e.entry_root, e) for e in self.entries])
     for eid in eid_map:
         elist = eid_map.getlist(eid)
         if len(elist) > 1:
             dup_id_map[eid] = elist
     if dup_id_map:
         raise ValueError('duplicate entry IDs detected: %r' % dup_id_map)
     self._call_custom_hook('post_validate')
Exemplo n.º 5
0
class EventEmitter2(object):
    def __init__(self):
        """
        """
        self.events = OrderedMultiDict()
        self.on = self.add_listener
        self.off = self.remove_listener
        # TODO: Add listener to remove

    def add_listener(self, event, listener, count=0):
        if (isinstance(listener, BuiltinFunctionType)):
            raise Exception("Invalid Listener: %s" % (str(listener)))
        _event = re.compile(event)
        _listener = {"handler": listener, "calls": 0, "calls_left": count}
        self.events.add(_event, _listener)
        return True

    def emit(self, event, kwargs):
        for pattern, listener in self.events.iteritems(multi=True):
            if pattern.match(event):
                if not listener["calls_left"]:
                    log.debug("Removing Listener: %s on Pattern: %s") % (
                        listener, pattern)
                    self.remove_listener(pattern, listener)
                listener["calls"] += 1
                listener["calls_left"] -= 1
                yield listener["handler"](**kwargs)

    def remove_listener(self, pattern, listener):
        pattern = re.compile(pattern)
        listeners = self.events.getlist(pattern)
        for pattern, _listener in self.events.iteritems(multi=True):
            if _listener['handler'] == listener:
                listener = _listener
                break
        listeners = self.events.getlist(pattern)
        listeners.remove(listener)
        if len(listeners):
            self.events.update({pattern: listeners})
        else:
            self.events._remove(pattern)
        return True

    def on_any(self, listener):
        raise NotImplementedError

    def off_any(self, listener):
        raise NotImplementedError

    def once(self, event, listener):
        self.add_listener(event, listener, left=1)

    def many(self, event, listener, left):
        self.add_listener(event, listener, left)
Exemplo n.º 6
0
 def validate(self):
     self._call_custom_hook('pre_validate')
     dup_id_map = {}
     eid_map = OMD([(e.entry_root, e) for e in self.entries])
     for eid in eid_map:
         elist = eid_map.getlist(eid)
         if len(elist) > 1:
             dup_id_map[eid] = elist
     if dup_id_map:
         raise ValueError('duplicate entry IDs detected: %r' % dup_id_map)
     self._call_custom_hook('post_validate')
Exemplo n.º 7
0
 def __init__(self, event_list=None, policy=None, marker=None):
     if event_list is None:
         event_list = []
     if policy is not None:
         for event in event_list:
             event._policy = policy
     self._marker = marker
     self._by_uuid = OrderedDict(
         zip(map(lambda e: e.uuid, event_list), event_list))
     self._by_message_id = OrderedMultiDict(
         zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values()))
Exemplo n.º 8
0
def read_instructions() -> OrderedMultiDict[str, str]:
    instructions = OrderedMultiDict()
    instruction_regex = re.compile(r"\s(\w)\s")
    with open(INPUT_FILE, "r") as f_handle:
        for line in f_handle:
            if line:
                key, value = instruction_regex.findall(line.rstrip())
                if key not in instructions:
                    instructions[key] = value
                else:
                    instructions.add(key, value)
    return instructions
Exemplo n.º 9
0
    def gen_connected_components(self):
        """Generate every connected oomponents"""
        # Initialize every ETH node with a distinct number
        connected_components = OrderedMultiDict(
            (nodeID, i) for i, (nodeID, n) in enumerate(node.items())
            if n['protocol'] == 'ETH')
        untraversed = self.switches.copy()

        while untraversed:
            # choos a switch as the root of the connected component
            root = untraversed.pop()
            # and set its number as the id
            ccID = connected_components[root]
            # switches of the same connected_components
            cc_switch = {root}
            while cc_switch:
                # current node
                cur = cc_switch.pop()
                for l in self.adj[cur]:
                    _, neighbor, _, _, _, _, _, _, _ = l
                    connected_components[neighbor] = ccID
                    if neighbor in untraversed:
                        cc_switch.add(neighbor)
                        untraversed.remove(neighbor)

        self.connected_components = connected_components
Exemplo n.º 10
0
def cs_scrape_properties(csid, props=None):
    '''Retrieve some of the experimental and predicted chemical properties that
    are not surfaced in the ChemSpider web API.'''
    try:
        r = requests.get('http://www.chemspider.com/Chemical-Structure.{0}.html'.format(csid))
        r.raise_for_status()
    except Exception as e:
        print(e)
        return None
    data = OMD([('CSID', csid)])
    doc = BeautifulSoup(r.text, 'lxml')
    # Experimental and predicted properties ("Experimental data" tab):
    props_tabs = doc.find(id='suppInfoTab')
    try:
        props_found = props_tabs.find_all(class_='user_data_property_name')
        for p in props_found:
            prop_name = p.get_text().strip(': \r\n')
            if props is not None and prop_name not in props:
                continue
            li = p.find_parent('li')
            values = li.find_all('td')
            for i in values:
                value = i.get_text().strip()
                data.add(prop_name, value)
    except AttributeError:
        pass
    # ACD/Labs predicted properties:
    acd_tab = doc.find(id='acdLabsTab')
    try:
        acd_props = acd_tab.find_all(class_='prop_title')
        for p in acd_props:
            prop_name = p.get_text().strip(': \r\n')
            v = p.find_next_sibling('td')
            value = v.get_text().strip()
            if props is not None and prop_name not in props:
                continue
            # Make sure all predicted properties are obviously named 
            if prop_name.startswith('ACD/') is False:
                prop_name = 'ACD/' + prop_name
            data.add(prop_name, value)
    except AttributeError:
        pass
    # EPI Suite results, as a blob to process later:
    #   Sometimes only returns part of the text, for some reason.
    if props is not None and 'EPI Suite' not in props:
        pass
    else:
        epi_tab = doc.find(id='epiTab')
        epi_blob = epi_tab.get_text().strip() if epi_tab else None
        data.add('EPI Suite', epi_blob)
    return data
Exemplo n.º 11
0
def analyze_samples():
    outcome = OrderedMultiDict()
    opcode = Opcode()
    methods = [
        func
        for func in getmembers(Opcode, predicate=isfunction)
        if func[0] != "__init__"
    ]
    count = 0
    for before, instruction, after in read_inputs():
        outcome.clear()
        for opcode_method in methods:
            if getattr(opcode, opcode_method[0])(before, instruction) == after:
                outcome.add(instruction[0], opcode_method[0])
        if len(outcome.getlist(instruction[0])) >= 3:
            count += 1
    #print(outcome)
    print(count)
Exemplo n.º 12
0
def epi_suite_values(epi_blob):
    '''Extracts some information as key-value pairs from EPI Suite output.
    Pretty rough (work in progress).'''
    data = OMD()
    try:
        lines = epi_blob.split('\n')
        for i in lines:
            j = i.strip()
            if j.startswith('Log Kow (') or j.startswith('Log BCF'):
                data.update(dict_from_line(i, '='))
            if j.startswith('Henrys LC') or\
               j.startswith('Log Koa (KOAWIN') or\
               j.startswith('Log Koa (experimental') or\
               j.startswith('Ready Biodegradability Prediction'):
                data.update(dict_from_line(i))
        if 'Fugacity' in epi_blob:
            model_table = epi_blob.split('Level III Fugacity Model:', 1)[1]
            data.add('Level III Fugacity Model', model_table)
    except:
        pass
    return data
Exemplo n.º 13
0
    def gen_connected_components(self):
        """
        Detect all the cut edges / cut endpoints (Tarjan Algorithm).
        Split the graph into different connected components.
        """

        # ----- temporary variables
        # temp: endpoints haven't been traversed
        self._tmp_untraversed = self.switches.copy()
        # temp: order of a node in the DFS
        self._tmp_order = defaultdict(lambda: INFTY)
        # temp: the lowest depth a node is connected to
        self._tmp_low = defaultdict(lambda: INFTY)
        # temp: parent of a node in the DFS
        self._tmp_parent = defaultdict(lambda: -1)

        # Initialize every ETH node with a distinct number
        node_connected_components = OrderedMultiDict(
            (nodeID, i) for i, (nodeID, n) in enumerate(node.items())
            if n['protocol'] == 'ETH')

        link_connected_components = OrderedMultiDict()

        # Initialize every ETH non-switch as type 2
        entity_type = OrderedMultiDict(
            (nodeID, 't2') for nodeID, n in node.items()
            if n['type'] != 'SWITCH' and n['protocol'] == 'ETH')

        while self._tmp_untraversed:
            # temp: all links of a connected components
            self._tmp_links = set()
            # temp: all endpoints of a connected components
            self._tmp_nodes = set()

            # ----- special links/endpoints
            ############################# Explanation ##########################
            """
            We categorize all the links and endpoints by 2 dimensions , so there
            are 2 * 2 = 4 categories in total. Suppose a link/endpoint is down,
            then there would be 4 possible results:

                                          New Routes are needed

                                            No      |       Yes  
                                                    |
                                    No      ---     |     Type I
            Some traffic will not                   |
            be reachable            ----------------+----------------
                                                    |
                                    Yes   Type II   |    Type III
                                                    |

            I'll use an example to explain these categories, and why we care
            about these scenarios:
            """
            
            ###################################################################
            #                                                                 #
            #                             A                                   #
            #                             |                                   #
            #                             |                                   #
            #                             |                                   #
            #                             1 ----- 2                           #
            #                              \     /                            #
            #                               \   /                             #
            #                                \ /                              #
            #                                 3                               #
            #                                 |                               #
            #                                 |                               #
            #                                 4                               #
            #                                / \                              #
            #                               /   \                             #
            #                              /     \                            #
            #                     C ----- 5       6 ------┐                   #
            #                             |       |       |                   #
            #                             |       |       B                   #
            #   A,B,C - DEVICES           |       |       |                   #
            #   1 ~ 8 - SWITCHES          7 ----- 8 ------┘                   #
            #                                                                 #
            ###################################################################

            """
            Type I: When the link/endpoint is down, new routes are needed, all
            traffic are still reachable
            - Endpoints: 2, 6, 7, 8
            - Links: 1-2, 1-3, 2-3, 4-5, 5-7, 7-8, 8-6, 6-4, 8-B, 6-B

            When one of these links/endpoints is down, the network will function
            as good, but traffic should take failover routes. Net Configurator
            will generate alternative routes for all of these scenarios.

            In the algorithm, such links/endpoints are not cut edges / cut 
            vertices.

            ------

            Type II: When the link/endpoint is down, every traffic will still
            take the primary route, while some of them will not be reachable.
            - Endpoints: A, B, C
            - Links: 3-4, 1-A, 5-C
            
            When one of these links/endpoints is down, some of the traffic will
            be dropped somewhere, but if a traffic is still reachable, then they
            will still take the primary route, so we will not generate new routes
            for these scenarios to reduce the ARL table size.

            In the algorithm, such links are cut edges; such endpoints are cut
            vertices, but not all cut vertices are type II endpoints. To be more
            precise, if an endpoint E connects to exactly one another endpoint, 
            or all the other endpoints that E connects to are not connected 
            w.r.t. any other endpoints except for E itself, then E is a type II
            endpoint.

            Note that in this example, if endpoint 5 is down, traffic between A 
            and B will not need a failover route, but 5 is not categorized as a
            type II endpoint, since 4 and 7 are connected w.r.t. some other
            links. Actually, we are not able to categorize 5 as type II to save
            memory, since connected components detection and link/node 
            categorization step is done before routes are calculated. Due to 
            that reason, 1 and 5 are not type II endpoints.

            I'm not sure if type II should exist. Although we can save memory
            in such scenarios, unreachable traffic will still take up the 
            bandwidth and resources of links/switches until they are dropped at
            the broken link/switch unless applications on top of IP Layer stops
            to send these traffic. Maybe we will decide to use more memory, but
            save runtime resources. There is a tradeoff.
            
            ------

            Type III: When the endpoint is down, some traffic will take failover
            routes while some may not and some traffic will be unreachable.
            - Endpoints: 1, 3, 4, 5

            When one of these endpoints is down, some traffic will be dropped
            and even some traffic are still reachable, they may need a failover
            route. In this scenario, we need to generate failover routes.

            In the algorithm, if an endpoint E connects to some endpoints that
            connect to each other w.r.t. links other than E, then E is a 
            type III endpoint.

            Note that no links are categorized as type III, since if a link is
            down, either all the devices are still connected in the same 
            connected component, or some are disconnected but we could use the 
            primary route and drop unreachable traffics somewhere.

            -----

            It will never happen that when a link/endpoint is down, no traffics
            are dropped and no failover route is needed. In this toy example,
            switch 2 is such an endpoint, but we still categorize it as type I
            because connected components detection and link/node categorization 
            step is done before routes are calculated.

            When we are calculating the shortest route tree, all the switches and
            links that not appears in the tree will be recorded. They will be a
            special part of type I that no failover routes will be generated for
            these links/switches. Endpoint 2, 7, 8 and links 1-2, 2-3, 5-7, 6-8,
            7-8, 8-B are such links/switches in this example.

            """

            # temp: non-cut links
            self._tmp_type1_links = set()
            # temp: cut links
            self._tmp_type2_links = set()

            # temp: non-cut endpoints
            self._tmp_type1_nodes = set()
            # temp: type 2 cut endpoints
            self._tmp_type2_nodes = set()
            # temp: type 3 cut endpoints
            self._tmp_type3_nodes = set()

            # order counter
            self.__order = 0 

            # randomly choose an endpoint as DFS root
            DFS_ROOT = self._tmp_untraversed.pop()

            self.bridge_update(DFS_ROOT)

            self._tmp_type1_links = self._tmp_links - self._tmp_type2_links

            ccID = node_connected_components[DFS_ROOT]

            node_connected_components.update({
                nodeID: ccID for nodeID in self._tmp_nodes
            })

            link_connected_components.update({
                linkID: ccID for linkID in self._tmp_links
            })

            entity_type.update({
                nodeID: 't1' for nodeID in self._tmp_type1_nodes
            })

            entity_type.update({
                nodeID: 't2' for nodeID in self._tmp_type2_nodes
            })

            entity_type.update({
                nodeID: 't3' for nodeID in self._tmp_type3_nodes
            })

            entity_type.update({
                linkID: 't1' for linkID in self._tmp_type1_links
            })

            entity_type.update({
                linkID: 't2' for linkID in self._tmp_type2_links
            })            

            self._tmp_untraversed -= self._tmp_nodes

        self.node_connected_components = node_connected_components
        self.link_connected_components = link_connected_components
        self.entity_type = entity_type
Exemplo n.º 14
0
class EventContext(object):
    def __init__(self, event_list=None, policy=None, marker=None):
        if event_list is None:
            event_list = []
        if policy is not None:
            for event in event_list:
                event._policy = policy
        self._marker = marker
        self._by_uuid = OrderedDict(
            zip(map(lambda e: e.uuid, event_list), event_list))
        self._by_id = OrderedMultiDict(
            zip(map(lambda e: e.id, self._by_uuid.values()),
                self._by_uuid.values()))

    def events(self):
        return list(self._by_uuid.values())

    def __iter__(self):
        return iter(self._by_uuid.values())

    def _set_marker(self, marker):
        self._marker = marker
        return self

    def filter(self, payload):
        if hasattr(payload, "id") and payload.id in self._by_id:
            events = self._by_id.getlist(payload.id)[:]
            return EventContext(events, marker=self._marker)
        else:
            return EventContext()

    def last(self, payload=None):
        if payload is None:
            return next(reversed(self._by_uuid.values()), None)
        elif hasattr(payload, "id"):
            return self._by_id.get(payload.id)
        else:
            raise RuntimeError(
                "EventContext.last() payload argument doesn't have an 'id' attribute"
            )

    def __bool__(self):
        return len(self._by_uuid) > 0

    __nonzero__ = __bool__

    def _marker_prefix_str(self):
        return self._marker.begin() if self._marker is not None else ""

    def _marker_suffix_str(self):
        return self._marker.end() if self._marker is not None else ""

    def _to_str(self):
        ret = ""
        if len(self._by_uuid.values()) > 1:
            ret += "["
        for i, event in enumerate(self._by_uuid.values()):
            ret += self._marker_prefix_str()
            ret += str(event)
            ret += self._marker_suffix_str()
            if i != (len(self._by_uuid.values()) - 1):
                ret += ","
        if len(self._by_uuid.values()) > 1:
            ret += "]"
        return ret

    def __str__(self):
        return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
Exemplo n.º 15
0
class ArsdkEventContext(object):

    def __init__(self, event_list=None, policy=None, marker=None):
        if event_list is None:
            event_list = []
        if policy is not None:
            for event in event_list:
                event._policy = policy
        self._marker = marker
        self._by_uuid = OrderedDict(
            zip(map(lambda e: e.uuid, event_list), event_list))
        self._by_message_id = OrderedMultiDict(
            zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values()))

    def events(self):
        return list(self._by_uuid.values())

    def __iter__(self):
        return iter(self._by_uuid.values())

    def _set_marker(self, marker):
        self._marker = marker
        return self

    def filter(self, message):
        if message.id in self._by_message_id:
            events = self._by_message_id.getlist(message.id)[:]
            return ArsdkEventContext(events, marker=self._marker)
        else:
            return ArsdkEventContext()

    def last(self, message=None):
        if message is None:
            return next(reversed(self._by_uuid.values()), None)
        else:
            return self._by_message_id.get(message.id)

    def __bool__(self):
        return len(self._by_uuid) > 0

    __nonzero__ = __bool__

    def _marker_prefix_str(self):
        return self._marker.begin() if self._marker is not None else ""

    def _marker_suffix_str(self):
        return self._marker.end() if self._marker is not None else ""

    def _to_str(self):
        ret = ""
        if len(self._by_uuid.values()) > 1:
            ret += "["
        for i, message_event in enumerate(self._by_uuid.values()):
            ret += self._marker_prefix_str()
            ret += str(message_event)
            ret += self._marker_suffix_str()
            if i != (len(self._by_uuid.values()) - 1):
                ret += ","
        if len(self._by_uuid.values()) > 1:
            ret += "]"
        return ret

    def __str__(self):
        return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
Exemplo n.º 16
0
 def __init__(self):
     """
     """
     self.events = OrderedMultiDict()
     self.on = self.add_listener
     self.off = self.remove_listener
Exemplo n.º 17
0
 def __init__(self):
     """
     """
     self.events = OrderedMultiDict()
     self.on = self.add_listener
     self.off = self.remove_listener