def raw_lines(self, node=None, style=None): if node == None: self.initialize() style = ChainMap({"fontstyle": "n"}) node = self.root for e in node.iterchildren(): if e.tag in ["html", "body", "pdf2xml"]: yield from self.raw_lines(e, style) continue if e.tag in ["b", "i"]: sstart, send = self.__class__.HTML_MARKUP[e.tag] yield sstart yield from self._texts(e, style.new_child({"fontstyle": e.tag})) yield send continue if e.tag in ["a"]: sstart, send = self.__class__.HTML_MARKUP[e.tag] yield sstart yield from self._texts(e, style) yield send continue if e.tag == "page": yield from self._proc_page(e, style) yield page_symbol continue #if e.tag == "text": # yield from self._proc_text(e, style) # continue yield e yield from self.raw_lines(e, style)
def __init__(self, **kwargs): # get dict fieldname:Field received as parameter parameters = { field: Field(value=Configuration.typed(field, value), type=Configuration.ALL_FIELDS[field].type) for field, value in kwargs.items() } # take parameters in priority, else the ALL_FIELDS dict prioritized_fields = ChainMap(parameters, Configuration.ALL_FIELDS) # save values as fields for varname, field in prioritized_fields.items(): value, ftype = field setattr(self, '_' + varname, ftype(value)) # access and setter definition def field_access(instance, field): return getattr(instance, '_' + field) def field_setter(instance, value, field): ftype = Configuration.ALL_FIELDS[field].type if field in self.UNMUTABLE_FIELDS: LOGGER.warning('CONFIG: The unmutable field ' + field + ' have been modified to ' + str(value)) elif field in self.GENERATED_FIELDS: LOGGER.warning('CONFIG: The generated field ' + field + ' have been modified to ' + str(value)) setattr(instance, '_' + field, ftype(value)) # regenerate generated data # add values as properties, eventually with a setter if field is mutable for varname in Configuration.ALL_FIELDS: new_prop = property(partial(field_access, field=varname)) setattr(Configuration, varname, new_prop) if varname in Configuration.MUTABLE_FIELDS: setter = new_prop.setter(partial(field_setter, field=varname)) setattr(Configuration, varname, setter) # add other data self.postprocess_data()
def load_settings(paths, default_config, oldsettings, setting_callbacks, setting_types, refresh_only): """ Load settings from the main config file. """ auto_settings, manual_settings\ = read_config(paths['config_file'], default_config) settings = ChainMap(auto_settings, manual_settings) def revert_setting(key): if key in oldsettings: reverted_value = oldsettings[key] else: reverted_value = default_config[key] if key in auto_settings: auto_settings[key] = reverted_value elif key in manual_settings: manual_settings[key] = reverted_value error = False # Make sure the settings aren't f****d up yo for key, value in settings.items(): if refresh_only and oldsettings.get(key) == value: continue # First make a simple check to see if the value is the right type if not valid_setting(key, value, setting_types): print('Invalid type for setting: "{}"'.format(key)) error = True revert_setting(key) # Then do a live update and see if things blow up try: update_runtime_setting(key, value, setting_callbacks) except SettingsError as e: print(str(e)) error = True revert_setting(key) error_text = 'Errors while reading the config. Check terminal output.' return settings, error_text if error else None
def store_selections(view, selections, replace=False, **settings): """ Save selections to storage. """ id = get_id(settings) if not replace: # Add old selections to selections, so as not to remove them selections = list(chain(selections, view.get_regions(id))) # Filter "settings" to only have "flags", "scope" and "icon" filtered = {"flags": parse_flags(**settings)} # Chain settings with preset to provide fallback settings = ChainMap(settings, presets[settings.get("preset", "")]) if "scope" in settings: filtered["scope"] = settings["scope"] if "icon" in settings: filtered["icon"] = settings["icon"] # Done! Finish up! # # Wrap the call in a timeout to appease adzenith who was kind enough # to open my first bug report at github.com/Veedrac/Sublime-Extras/issues/1. # This should make sure that you can switch panes with the mouse. sublime.set_timeout(lambda: view.add_regions(id, selections, **filtered), 0)
def more(): values = ChainMap() values['x'] = 1 print(values) values = values.new_child() values['x'] = 2 print(values) values = values.new_child() values['x'] = 3 print(values, values['x'], values.parents['x'], values.parents.parents['x'])
def fromDocAndFlags(document, flags): """ @param document: input document @param flags: additional API parameters @return: S2ApiInput corresponding to given document and flags """ docDict = {k: v for k, v in document._asdict().items() if v} data = ChainMap(docDict, flags) inputDict = {fldName : data.get(fldName, None) for fldName in S2ApiInput._fields} return S2ApiInput(**inputDict)
def simple_cm(): from collections import ChainMap c = ChainMap() c['one'] = 1 c['two'] = 2 cc = c.new_child() cc['one'] = 'one' return c, cc
def get_raw_results(self): columns = [] response_cols = [] for p, responses in self._stored_responses.items(): for i, response in responses.items(): response = ChainMap( response, {'participant_id': p, 'active_item_id': i}) if not columns: columns.extend(response.keys()) response_cols.append([response[c] for c in columns]) return columns, response_cols
def test_bool(self): from collections import ChainMap c = ChainMap() assert not(bool(c)) c['one'] = 1 c['two'] = 2 assert bool(c) cc = c.new_child() cc['one'] = 'one' assert cc
def __contains__(self, k): """Check whether a variable has been assigned to. This is **not** the same kind of element-of as described in the class documentation. Args: k (str): The name of the variable to check. Returns: bool: Whether or not the variable has been assigned to. """ chain = ChainMap(self.scopes, self.globals) return chain.__contains__(k)
def instantiate_related_objects(self, related_model, related_objects, meta_attrs): new_objects = [] for obj_attrs in related_objects: new_obj = related_model() input_attrs = ChainMap(obj_attrs, meta_attrs) for field_name, field_value in input_attrs.items(): f = self.interpolate(field_name, meta_attrs) setattr(new_obj, field_name, f(field_value)) new_objects.append(new_obj) return new_objects
def __getitem__(self, k): """Look up a variable. Args: k (str): The name of the variable to look up. Returns: LispVal: The value assigned to the variable. Raises: KeyError: If the variable has not been assigned to. """ chain = ChainMap(self.scopes, self.globals) return chain.__getitem__(k)
def expand_rows(self, apply_extended=True): """Generate CIB rows by expanding all CIBs pointing to current CIB """ paths = self.resolve_graph() # for storing expanded rows rows = [] for path in paths: expanded_properties = (self.cib[uid].expand() for uid in path) for pas in itertools.product(*expanded_properties): chain = ChainMap(*pas) # For debugging purposes, add the path list to the chain. # Store as string to preserve path order (NEAT properties are not ordered). dbg_path = '<<'.join(uid for uid in path) # insert at position 0 to override any existing entries # chain.maps.insert(0, PropertyArray(NEATProperty(('cib_uids', dbg_path)))) # convert back to normal PropertyArrays row = PropertyArray(*(p for p in chain.values())) row.meta['cib_uids'] = dbg_path rows.append(row) if not apply_extended: return rows if not self.cib.extenders: # no extender CIB nodes loaded return rows # TODO optimize extended_rows = rows.copy() for entry in rows: # TODO take priorities into account # iterate extender cib_nodes for uid, xs in self.cib.extenders.items(): for pa in xs.expand(): if xs.match_entry(entry): entry_copy = copy.deepcopy(entry) chain = ChainMap(pa, entry_copy) new_pa = PropertyArray(*(p for p in chain.values())) try: del new_pa['uid'] except KeyError: pass extended_rows.append(new_pa) return extended_rows
def __init__(self, schema): self.count = -1 self.lookup = ChainMap() self.type_of_var = {} self.schema = schema self.constants = set() self.backrefs = build_links(schema)
def __init__(self, functions, name="module"): self.vars = ChainMap() self.vars.update({fn.instructions[0][1]: fn for fn in functions}) # List of Python modules to search for external decls external_libs = [ 'math', 'os' ] self.external_libs = [ __import__(name) for name in external_libs]
def __init__(self): self.symtab = ChainMap({ 'int': types.IntType, 'float': types.FloatType, 'string': types.StringType, 'bool': types.BoolType, })
def test_fromkeys(self): from collections import ChainMap keys = 'a b c'.split() c = ChainMap.fromkeys(keys) assert len(c) == 3 assert c['a'] == None assert c['b'] == None assert c['c'] == None
def __init__(self, *, settings=None, device=None): self.settings = settings or TextViewSettings() self._metrics = Qt.QFontMetricsF(self.settings.q_font) self.device = device self.reset() self._base_attrs = {} self._cur_lc_attrs = {} self._cur_attrs = {} self._attrs = ChainMap(self._cur_attrs, self._cur_lc_attrs, self._base_attrs)
class Module(object): def __init__(self, name, fullname, parent=None, reader=None): self.name = name self.fullname = fullname self.parent = parent self.reader = reader self.files = OrderedDict() self.reset() @property def package_name(self): return self.name def fulladdress(self, name): return "{}.{}".format(self.fullname, name) def read_file(self, name, file): self.files[name] = self.reader.read_file(file, parent=self) def normalize(self): self.reset() for file in self.files.values(): file.normalize() self.new_child(file.members) def reset(self): self.members = ChainMap() def new_child(self, item): self.members = self.members.new_child(item) def __getitem__(self, name): return self.members[name] def get(self, name, default=None): return self.members.get(name, default) def __contains__(self, name): return name in self.members @property def world(self): return self.parent
def __init__(self, loop, *, error_table=None, translation_table=None): super().__init__(loop, translation_table=translation_table) self.calls = {} self.prefix = self.REQ_PREFIX.pack(os.getpid() % 0x10000, random.randrange(0x10000)) self.counter = 0 if error_table is None: self.error_table = _default_error_table else: self.error_table = ChainMap(error_table, _default_error_table)
def __init__(self): """ mock knowledge base, use triple store ? """ self.base = ChainMap(self.who ,self.what ,self.where ,self.when ,self.why ,self.defaults ,self.uq )
class ScopeStack(object): def __init__(self): self.data = ChainMap({}) self.special_forms = {} def __contains__(self, key): return key in self.data def set(self, key, value, special_form=False): if not special_form: self.data[key] = value else: self.special_forms[key] = value if not self.validate(): raise RuntimeError("Special form overwriten") def unset(self, key): del self.data[key] def is_empty(self): return len(self.data) == 0 def new_scope(self): self.data = self.data.new_child() def drop_scope(self): self.data = self.data.parents def get_scope_identifiers(self, root=False): first_level = self.data.maps[0] if len(self.data.maps) > 0 else {} merged_stmts = list(first_level.values()) if root: merged_stmts = list(self.special_forms.values()) + merged_stmts return sorted(merged_stmts, key=lambda x: x.value) def validate(self): normal_scope = set(self.data.keys()) special_form_scope = set(self.special_forms.keys()) return len(normal_scope.intersection(special_form_scope)) == 0
def __new__(self, name, bases, attrs): meta = ChainMap({}) used = set() for b in bases: if hasattr(b, "meta"): meta = meta.new_child(b.meta) for cls in b.mro(): used.add(cls) else: for cls in reversed(b.mro()): if cls in used: continue used.add(cls) if not hasattr(cls, "Meta"): continue meta = meta.new_child(cls.Meta.__dict__) if "Meta" in attrs: meta = meta.new_child(attrs["Meta"].__dict__) attrs.update({"meta": meta}) return super().__new__(self, name, bases, attrs)
def __init__(self,method='mcmc'): # self.variable_list={} self.scalar_list={} self.element_list={} self.follower_list={} self.dependent_list={} # self.matrix_list={} # self.free_parameter_list={} self.variable_list=ChainMap( self.scalar_list,self.element_list,self.follower_list,self.dependent_list ) self.free_parameter_list=ChainMap( self.scalar_list,self.element_list ) self.block_list={} self.method=method.lower()
def test_chain_map(): from collections import ChainMap ChainMap() cm = ChainMap({"a": 1, "b": 2}) assert cm.maps is not None assert cm.new_child().maps is not None assert cm.new_child({"c": 3}).maps is not None assert cm.parents.maps is not None cm["d"] = 4 del cm["a"] assert cm["a"] is None assert cm["b"] == 2 assert cm["d"] == 4 assert list(iter(cm)) == ["b", "d", "c"] assert len(cm) == 3
class SymbolTable(object): ''' Class representing a symbol table. It should provide functionality for adding and looking up nodes associated with identifiers. ''' def __init__(self): self.table = ChainMap() self.current_scope = self.table self.root_scope = self.table self.type_objects = { int: gonetype.int_type, float: gonetype.float_type, str: gonetype.string_type, bool: gonetype.bool_type, 'int': gonetype.int_type, 'float': gonetype.float_type, 'string': gonetype.string_type, 'bool': gonetype.bool_type } def add(self, symbol, data): self.current_scope[symbol] = data def get(self, symbol): if symbol in self.current_scope: return self.current_scope[symbol] return None def push_scope(self): self.current_scope = self.table.new_child() return self.current_scope def pop_scope(self): self.current_scope = self.current_scope.parents return self.current_scope def pprint(self): print("{}top".format("-" * 10)) for symbol in self.table: print("{}: {}".format(symbol, self.table.get(symbol))) print("-" * 10)
def combine_map(): a = {'x': 1, 'z': 3 } b = {'y': 2, 'z': 4 } c = ChainMap(a,b) print(c['x']) # Outputs 1 (from a) print(c['y']) # Outputs 2 (from b) print(c['z']) # Outputs 3 (from a) print(len(c)) print(list(c.keys())) print(list(c.values())) c['z'] = 10 c['w'] = 40 del c['x'] print(a) # del c['y'] values = ChainMap() values['x'] = 1 # Add a new mapping values = values.new_child() values['x'] = 2 # Add a new mapping values = values.new_child() values['x'] = 3 print(values) print(values['x']) # Discard last mapping values = values.parents print(values['x']) # Discard last mapping values = values.parents print(values['x']) print(values)
def prepare(func=None, param_types=dict()): if func is None: from __main__ import main as func param_types = ChainMap(param_types, getattr(func, "param_types", dict())) try: sig = signature(func) except (ValueError, TypeError): return (func, None, dict(), param_types) keyword_kinds = (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) keywords = OrderedDict((param.name, param) for param in sig.parameters.values() if param.kind in keyword_kinds) # Explicit set() construction to work around Python 2's keys() lists missing = set(param_types.keys()).difference(sig.parameters.keys()) if missing: missing = ", ".join(map(repr, missing)) msg = "{}() missing parameters: {}".format(func.__name__, missing) raise TypeError(msg) return (func, sig, keywords, param_types)
def ch1_20(): ''' 1.20 将多个映射合并为单个映射 使用collections ChainMap解决 ''' print("\nch1_20:") from collections import ChainMap a={'x':1,'z':3} b={'y':2,'z':4} c = ChainMap(a,b) #使用原始的字典 print(c['x'],c['y'],c['z']) print(list(c.keys()),list(c.values())) #用新建字典代替 merged = dict(b) merged.update(a) print(merged['x'],merged['y'],merged['z'])
def __init__(self): self.table = ChainMap() self.current_scope = self.table self.root_scope = self.table self.type_objects = { int: gonetype.int_type, float: gonetype.float_type, str: gonetype.string_type, bool: gonetype.bool_type, 'int': gonetype.int_type, 'float': gonetype.float_type, 'string': gonetype.string_type, 'bool': gonetype.bool_type }
def __init__(self, source, target, model, connection_dict): """Initialize Connection object from model and overrides. Initialize the self.params and self.nest_params attributes, and all the other attributes as well :) Args: source (Layer): source Layer object target (Layer): target Layer object model (ConnectionModel): ConnectionModel object. Provide base 'params' and 'nest_params' parameter dictionaries. connection_dict (dict): Dictionary defining the connection. The dictionary should have the form described in the class docstring. In particular, it may contain the following keys: params (dict): "non-nest" parameter dictionary. Combined in a ChainMap with `model.params`. All recognized parameters are listed in global variable `NON_NEST_CONNECTION_PARAMS`. nest_params (dict): Parameters that may be passed to the NEST kernel. Combined in a ChainMap with model.nest_params. No parameter listed in global variable `NON_NEST_CONNECTION_PARAMS` should be present in this variable. """ ## # Check the params and nest_params dictionaries and ChainMap them with # the ConnectionModel params and nest_params params = connection_dict.get('params', {}) nest_params = connection_dict.get('nest_params', {}) assert all([key in NON_NEST_CONNECTION_PARAMS for key in params.keys()]), \ (f'Unrecognized parameter in connection: {connection_dict}.' f'\nRecognized parameters: {NON_NEST_CONNECTION_PARAMS.keys()}') assert not any([key in NON_NEST_CONNECTION_PARAMS for key in nest_params.keys()]), \ (f'Reserved nest parameter in connection: {connection_dict}' f'\"Non-nest reserved parameters: {NON_NEST_CONNECTION_PARAMS.keys()}') self.params = dict(ChainMap(params, model.params)) self.nest_params = dict(ChainMap(nest_params, model.nest_params)) super().__init__(model.name, self.params) ## # Define the source/target population attributes self.model = model self.source = source self.source_population = connection_dict.get('source_population', None) self.target = target self.target_population = connection_dict.get('target_population', None) # By default, we consider the driver to be the source self.driver = 'source' self.driver_layer = self.source self.driver_population = self.source_population ## # Synapse model is retrieved either from nest_params or UnitConns. # The synapse model used in NEST might have a different name since we # need to change the default parameters of a synapse to specify the # weight recorder self._synapse_model = None self._nest_synapse_model = None ## # Synapse label is set in the defaults of the nest synapse model in # `create_nest_synapse_model` and can be used to query effectively the # connections of a certain preojection. self._synapse_label = None # Initialize the recorders self.recorders = [ ConnectionRecorder(recorder_name, recorder_params) for recorder_name, recorder_params in self.params['recorders'].items() ] assert len(self.recorders) < 2 # Only a single recorder type so far... self.check()
for word in words: counts_dict[word[0]] = counts_dict.get(word[0], 0) + 1 print(counts_dict) # yet another way counts_dict = {} counts_dict = defaultdict(int) for word in words: counts_dict[word[0]] = counts_dict[word[0]] + 1 print(counts_dict) # using chainmap d = {'a': 1, 'b': 2} d1 = {'a': 11, 'b': 22} d2 = {'a': 33, 'c': 44} print("chain map is") print(ChainMap(d, d1, d2)) # List comprehension and generator expressions. a = (el for el in range(4)) # creates a generator print(a) # Output: <generator object <genexpr> at 0x100b449d0> # python popitem
# Chain Maps from collections import ChainMap default_connection = {'host': 'localhost', 'port': 4567} connection = {'port': 5678} conn = ChainMap(connection, default_connection) conn['port'] conn['host']
def __init__(self, app_state: collections.ChainMap) -> None: self.id = uuid.uuid4() self._state = app_state.new_child()
# FileName : Chapter1_20.py ''' 合并多个字典或映射 关键点: collections.ChainMap:合并多个字典但不产生新的字典数据,所以增删改会影响源字典 dict.update:合并多个字典并产生新的字典,数据独立存储 ''' from collections import ChainMap if __name__ == '__main__': a = {'x': 1, 'z': 3} b = {'y': 2, 'z': 4} c = ChainMap(a, b) print(a) print(b) print(c) print(len(c)) print(list(c.keys())) print(list(c.values())) c['z'] = 10 c['w'] = 40 del c['x'] b['y'] = 20 print(a) print(b) print(c) values = ChainMap() values['x'] = 1 values = values.new_child() values['x'] = 2
def _cat(maps: list): # make callbacks' return to a single map maps = [m for m in maps if m is not None] return dict(ChainMap(*maps))
def get_document(self): hdr = self._get_doc_header() bdy = self._get_doc_body() return [ChainMap(*[hdr, dct]) for dct in bdy]
import json from collections import ChainMap file_name = 'south-park.json' final_file = 'lines-by-character.json' with open(file_name, "r") as read_file: data = json.load(read_file) all_characters = list(set([line['character'] for line in data])) # code smell all_characters.sort() lines_by_character = [] for character in all_characters: character_lines = [] print(f'Current character: {character}') for line in data: if line['character'] == character: character_lines.append(line['line']) lines_by_character.append({character: character_lines}) print(f'{character} done.') # print(lines_by_character[0]) data = dict(ChainMap(*lines_by_character)) json_string = json.dumps(data, indent=4) with open(final_file, "w") as outfile: outfile.write(json_string)
def create_alias_replacement( self, imported_symbols: Dict[TypeAnnotationNode, TypeAnnotationNode] ) -> AliasReplacementVisitor: return AliasReplacementVisitor( ChainMap(imported_symbols, self.__project_specific_aliases, self.__aliases))
class TILookup: """Threat Intel observable lookup from providers.""" def __init__( self, primary_providers: Optional[List[TIProvider]] = None, secondary_providers: Optional[List[TIProvider]] = None, providers: Optional[List[str]] = None, ): """ Initialize TILookup instance. Parameters ---------- primary_providers : Optional[List[TIProvider]], optional Primary TI Providers, by default None secondary_providers : Optional[List[TIProvider]], optional Secondary TI Providers, by default None providers: Optional[List[str]], optional List of provider names to load, by default all available providers are loaded. To see the list of available providers call `TILookup.list_available_providers()`. Note: if primary_provides or secondary_providers is specified This will override the providers list. """ self._providers: Dict[str, TIProvider] = {} self._secondary_providers: Dict[str, TIProvider] = {} self._providers_to_load = providers if primary_providers: for prov in primary_providers: self.add_provider(prov, primary=True) if secondary_providers: for prov in secondary_providers: self.add_provider(prov, primary=False) if not (primary_providers or secondary_providers): self._load_providers() self._all_providers = ChainMap(self._secondary_providers, self._providers) @property def loaded_providers(self) -> Dict[str, TIProvider]: """ Return dictionary of loaded providers. Returns ------- Dict[str, TIProvider] [description] """ return self._all_providers # type: ignore @property def provider_status(self) -> Iterable[str]: """ Return loaded provider status. Returns ------- Iterable[str] List of providers and descriptions. """ prim = [ f"{prov_name} - {prov.description} (primary)" for prov_name, prov in self._providers.items() ] sec = [ f"{prov_name} - {prov.description} (secondary)" for prov_name, prov in self._secondary_providers.items() ] return prim + sec @property def configured_providers(self) -> List[str]: """ Return a list of avaliable providers that have configuration details present. Returns ------- List[str] List of TI Provider classes. """ prim_conf = list(self._providers.keys()) sec_conf = list(self._secondary_providers.keys()) return prim_conf + sec_conf @property def available_providers(self) -> List[str]: """ Return a list of builtin providers. Returns ------- List[str] List of TI Provider classes. """ return self._get_available_providers() @classmethod def _get_available_providers(cls): providers = [] for provider_name in dir(tiproviders): provider_class = getattr(tiproviders, provider_name, None) if not (provider_class and isclass(provider_class)): continue # if it is a class - we only want to show concrete classes # that are sub-classes of TIProvider if issubclass(provider_class, tiproviders.TIProvider) and not bool( getattr(provider_class, "__abstractmethods__", False)): providers.append(provider_class.__name__) return providers @classmethod def list_available_providers( cls, show_query_types=False, as_list: bool = False) -> Optional[List[str]]: # type: ignore """ Print a list of builtin providers with optional usage. Parameters ---------- show_query_types : bool, optional Show query types supported by providers, by default False as_list : bool, optional Return list of providers instead of printing to stdout. Note: if you specify `show_query_types` this will be printed irrespective of this parameter setting. Returns ------- Optional[List[str]] A list of provider names (if `return_list=True`) """ providers = [] for provider_name in cls._get_available_providers(): provider_class = getattr(tiproviders, provider_name, None) if not as_list: print(provider_name) providers.append(provider_name) if show_query_types: provider_class.usage() if as_list: return providers return None def provider_usage(self): """Print usage of loaded providers.""" print("Primary providers") print("-----------------") if self._providers: for prov_name, prov in self._providers.items(): print(f"\nProvider class: {prov_name}") prov.usage() else: print("none") print("\nSecondary providers") print("-------------------") if self._secondary_providers: for prov_name, prov in self._secondary_providers.items(): print(f"\nProvider class: {prov_name}") prov.usage() else: print("none") @classmethod def reload_provider_settings(cls): """Reload provider settings from config.""" reload_settings() print( "Settings reloaded. Use reload_providers to update settings", "for loaded providers.", ) def reload_providers(self): """ Reload providers based on currrent settings in config. Parameters ---------- clear_keyring : bool, optional Clears any secrets cached in keyring, by default False """ self.reload_provider_settings() self._load_providers() def _load_providers(self): """Load provider classes based on config.""" prov_settings = get_provider_settings() for provider_entry, settings in prov_settings.items(): # Allow overriding provider name to use another class provider_name = settings.provider or provider_entry if self._providers_to_load and provider_name not in self._providers_to_load: continue provider_class: TIProvider = getattr(sys.modules[__name__], provider_name, None) if not provider_class: warnings.warn( f"Could not find provider class for {provider_name} " + f"in config section {provider_entry}") continue # instantiate class sending args from settings to init try: provider_instance = provider_class(**(settings.args)) except MsticpyConfigException as mp_ex: # If the TI Provider didn't load, raise an exception raise MsticpyUserConfigError( f"Could not load TI Provider {provider_name}", *mp_ex.args, "To avoid loading this provider please use the 'providers' parameter" + " to TILookup() to specify which providers to load.", title="TIProvider configuration error", help_uri= "https://msticpy.readthedocs.io/en/latest/data_acquisition/" + "TIProviders.html#configuration-file", ) from mp_ex # set the description from settings, if one is provided, otherwise # use class docstring. provider_instance.description = (settings.description or provider_instance.__doc__) self.add_provider(provider=provider_instance, name=provider_name, primary=settings.primary) def add_provider(self, provider: TIProvider, name: str = None, primary: bool = True): """ Add a TI provider to the current collection. Parameters ---------- provider : TIProvider Provider instance name : str, optional The name to use for the provider (overrides the class name of `provider`) primary : bool, optional "primary" or "secondary" if False, by default "primary" """ if not name: name = provider.__class__.__name__ if primary: self._providers[name] = provider else: self._secondary_providers[name] = provider # pylint: disable=too-many-arguments def lookup_ioc( self, observable: str = None, ioc_type: str = None, ioc_query_type: str = None, providers: List[str] = None, prov_scope: str = "primary", **kwargs, ) -> Tuple[bool, List[Tuple[str, LookupResult]]]: """ Lookup single IoC in active providers. Parameters ---------- observable : str IoC observable (`ioc` is also an alias for observable) ioc_type : str, optional One of IoCExtract.IoCType, by default None If none, the IoC type will be inferred ioc_query_type: str, optional The ioc query type (e.g. rep, info, malware) providers: List[str] Explicit list of providers to use prov_scope : str, optional Use "primary", "secondary" or "all" providers, by default "primary" kwargs : Additional arguments passed to the underlying provider(s) Returns ------- Tuple[bool, List[Tuple[str, LookupResult]]] The result returned as a tuple(bool, list): bool indicates whether a TI record was found in any provider list has an entry for each provider result """ if not observable and "ioc" in kwargs: observable = kwargs["ioc"] if not observable: raise ValueError("observable or ioc parameter must be supplied.") result_list: List[Tuple[str, LookupResult]] = [] selected_providers = self._select_providers(providers, prov_scope) if not selected_providers: raise MsticpyUserConfigError( _NO_PROVIDERS_MSSG, title="No Threat Intel Provider configuration found.", help_uri=_TI_HELP_URI, ) ioc_type = ioc_type or TIProvider.resolve_ioc_type(observable) for prov_name, provider in selected_providers.items(): provider_result: LookupResult = provider.lookup_ioc( ioc=observable, ioc_type=ioc_type, query_type=ioc_query_type, **kwargs) result_list.append((prov_name, provider_result)) overall_result = any(res.result for _, res in result_list) return overall_result, result_list def lookup_iocs( self, data: Union[pd.DataFrame, Mapping[str, str], Iterable[str]], obs_col: str = None, ioc_type_col: str = None, ioc_query_type: str = None, providers: List[str] = None, prov_scope: str = "primary", **kwargs, ) -> pd.DataFrame: """ Lookup a collection of IoCs. Parameters ---------- data : Union[pd.DataFrame, Mapping[str, str], Iterable[str]] Data input in one of three formats: 1. Pandas dataframe (you must supply the column name in `obs_col` parameter) 2. Mapping (e.g. a dict) of [observable, IoCType] 3. Iterable of observables - IoCTypes will be inferred obs_col : str, optional DataFrame column to use for observables, by default None ioc_type_col : str, optional DataFrame column to use for IoCTypes, by default None ioc_query_type: str, optional The ioc query type (e.g. rep, info, malware) providers: List[str] Explicit list of providers to use prov_scope : str, optional Use "primary", "secondary" or "all" providers, by default "primary" kwargs : Additional arguments passed to the underlying provider(s) Returns ------- pd.DataFrame DataFrame of results """ result_list: List[pd.DataFrame] = [] selected_providers = self._select_providers(providers, prov_scope) if not selected_providers: raise MsticpyUserConfigError( _NO_PROVIDERS_MSSG, title="No Threat Intel Provider configuration found.", help_uri=_TI_HELP_URI, ) for prov_name, provider in selected_providers.items(): provider_result = provider.lookup_iocs( data=data, obs_col=obs_col, ioc_type_col=ioc_type_col, query_type=ioc_query_type, **kwargs, ) if provider_result is None or provider_result.empty: continue if not kwargs.get("show_not_supported", False): provider_result = provider_result[ provider_result["Status"] != TILookupStatus.not_supported.value] if not kwargs.get("show_bad_ioc", False): provider_result = provider_result[ provider_result["Status"] != TILookupStatus.bad_format.value] provider_result["Provider"] = prov_name result_list.append(provider_result) if not result_list: print("No IoC matches") return pd.concat(result_list, sort=False) @staticmethod def result_to_df( ioc_lookup: Tuple[bool, List[Tuple[str, LookupResult]]]) -> pd.DataFrame: """ Return DataFrame representation of IoC Lookup response. Parameters ---------- ioc_lookup : Tuple[bool, List[Tuple[str, LookupResult]]] Output from `lookup_ioc` Returns ------- pd.DataFrame The response as a DataFrame with a row for each provider response. """ return (pd.DataFrame({ r_item[0]: pd.Series(attr.asdict(r_item[1])) for r_item in ioc_lookup[1] }).T.rename(columns=LookupResult.column_map()).drop("SafeIoc", axis=1)) def _select_providers( self, providers: List[str] = None, prov_scope: str = "primary") -> Dict[str, TIProvider]: """ Return required subset of providers. Parameters ---------- providers : List[str], optional Explicit list of provider names, by default None prov_scope : str, optional Provider scope, by default "primary" Other values are "all" and "secondary" Returns ------- Dict[str, TIProvider] Dictionary of provider names and instances. """ if providers: selected_providers = { prov_name: prov for prov_name, prov in self._all_providers.items() if prov_name in providers } else: if prov_scope == "all": selected_providers = self._all_providers # type: ignore elif prov_scope == "primary": selected_providers = self._providers else: selected_providers = self._secondary_providers return selected_providers @staticmethod def browse_results(data: pd.DataFrame, severities: Optional[List[str]] = None, **kwargs): """ Return TI Results list browser. Parameters ---------- data : pd.DataFrame TI Results data from TIProviders severities : Optional[List[str]], optional A list of the severity classes to show. By default these are ['warning', 'high']. Pass ['information', 'warning', 'high'] to see all results. Other Parameters ---------------- kwargs : passed to SelectItem constuctor. Returns ------- SelectItem SelectItem browser for TI Data. """ return browse_results(data=data, severities=severities, **kwargs)
def combine_spaces(spaces: List[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: return [dict(ChainMap(*combination)) for combination in product(*spaces)]
def __init__( self, access_key_id=None, account_name=None, cleanup=None, config=None, description=None, distro_name=None, early_exit=None, history_log=None, image_id=None, inject=None, instance_type=None, log_level=None, no_default_test_dirs=None, cloud_config=None, region=None, results_dir=None, running_instance_id=None, secret_access_key=None, security_group_id=None, ssh_key_name=None, ssh_private_key_file=None, ssh_user=None, subnet_id=None, test_dirs=None, test_files=None, timeout=None, collect_vm_info=None, enable_secure_boot=None, enable_uefi=None, log_callback=None, prefix_name=None, retry_count=None ): """Initialize EC2 cloud framework class.""" super(EC2Cloud, self).__init__( 'ec2', cleanup, config, description, distro_name, early_exit, history_log, image_id, inject, instance_type, log_level, no_default_test_dirs, cloud_config, region, results_dir, running_instance_id, test_dirs, test_files, timeout, collect_vm_info, ssh_private_key_file, ssh_user, subnet_id, enable_secure_boot, enable_uefi, log_callback, prefix_name, retry_count ) # Get command line values that are not None cmd_line_values = self._get_non_null_values(locals()) self.zone = None self.account_name = account_name if not self.account_name: self.logger.debug( 'No account provided. To use the EC2 config file an ' 'account name is required.' ) if not self.region: raise EC2CloudException( 'Region is required to connect to EC2.' ) elif self.region[-1].isalpha(): self.zone = self.region self.region = self.region[:-1] config_file = self.cloud_config or EC2_CONFIG_FILE ec2_config = {} try: ec2_config = ipa_utils.get_config_values( config_file, ''.join(['region-', self.region]), ''.join(['account-', self.account_name]) ) self.logger.debug( 'Using EC2 config file: %s' % config_file ) except Exception: self.logger.debug( 'EC2 config file not found: %s' % config_file ) self.ec2_config = defaultdict( lambda: None, ChainMap(cmd_line_values, ec2_config, self.ipa_config) ) self.access_key_id = self.ec2_config['access_key_id'] self.secret_access_key = self.ec2_config['secret_access_key'] self.security_group_id = self.ec2_config['security_group_id'] self.ssh_key_name = self.ec2_config['ssh_key_name'] self.subnet_id = self.ec2_config['subnet_id'] self.ssh_user = ( cmd_line_values.get('ssh_user') or ec2_config.get('user') or self.ipa_config.get('ssh_user') or EC2_DEFAULT_USER ) self.ssh_private_key_file = ( cmd_line_values.get('ssh_private_key_file') or ec2_config.get('ssh_private_key') or self.ipa_config.get('ssh_private_key_file') ) if not self.ssh_private_key_file: raise EC2CloudException( 'SSH private key file is required to connect to instance.' ) else: self.ssh_private_key_file = os.path.expanduser( self.ssh_private_key_file )
def installed(self): """Provides a view into all installed mods (including dependencies).""" return ChainMap(self.mods, self.dependencies)
def eval(x, env=None): """ Avalia expressão no ambiente de execução dado. """ # Cria ambiente padrão, caso o usuário não passe o argumento opcional "env" if env is None: env = ChainMap({}, global_env) # Avalia tipos atômicos if isinstance(x, Symbol): return env[x] elif isinstance(x, (int, float, bool, str)): return x # Avalia formas especiais e listas head, *args = x # Comando (if <test> <then> <other>) # Ex: (if (even? x) (quotient x 2) x) if head == Symbol.IF: (condition, then, alternative) = args expression = (then if eval(condition, env) else alternative) return eval(expression, env) # Comando (define <symbol> <expression>) # Ex: (define x (+ 40 2)) elif head == Symbol.DEFINE: variable, value_or_expression = args new_thing = eval(value_or_expression, env) env[Symbol(variable)] = new_thing return None # Comando (quote <expression>) # (quote (1 2 3)) elif head == Symbol.QUOTE: result = [] arguments = args[0] if isinstance(arguments, list): for x in args[0]: if isinstance(x, (int, float, bool, str)): result.append(eval(x, env)) else: result.append(Symbol(x)) else: return arguments return result # Comando (let <expression> <expression>) # (let ((x 1) (y 2)) (+ x y)) elif head == Symbol.LET: sub_env = ChainMap( {}, global_env) # pegar as funções sem ter que copiar todo o env declarations, expr = args for declaration in declarations: eval([Symbol.DEFINE, declaration[0], declaration[1]], sub_env) result = eval(expr, sub_env) return result # Comando (lambda <vars> <body>) # (lambda (x) (+ x 1)) elif head == Symbol.LAMBDA: if len(args) == 1: print(args[0]) parameters, expr = args[0] else: parameters, expr = args result = None print("parameters: ", parameters) if any( isinstance(parameter, (float, int, bool)) for parameter in parameters): raise TypeError local_ctx = ChainMap({}, global_env) def new_fun(*arguments): arguments = list(arguments) for parameter_number in range(len(parameters)): if len(arguments) > 0: local_ctx[parameters[parameter_number]] = arguments[ parameter_number] else: local_ctx[parameters[parameter_number]] = arguments return eval(expr, local_ctx) return new_fun # Lista/chamada de funções # (sqrt 4) elif head == Symbol.ADD: x, y = args return eval(x, env) + eval(y, env) elif head == Symbol.SUB: x, y = args return eval(x, env) - eval(y, env) else: env_function = eval(head, env) arguments = (eval(arg, env) for arg in x[1:]) return env_function(*arguments)
def from_dict(cls, d, href=None, root=None): id = d['id'] geometry = d['geometry'] bbox = d['bbox'] properties = d['properties'] stac_extensions = d.get('stac_extensions') collection_id = None if 'collection' in d.keys(): collection_id = d['collection'] datetime = properties.get('datetime') if datetime is None: raise STACError( 'Item dict is missing a "datetime" property in the "properties" field' ) datetime = dateutil.parser.parse(datetime) item = Item(id=id, geometry=geometry, bbox=bbox, datetime=datetime, properties=properties, stac_extensions=stac_extensions, collection=collection_id) for l in d['links']: item.add_link(Link.from_dict(l)) for k, v in d['assets'].items(): asset = Asset.from_dict(v) asset.set_owner(item) item.assets[k] = asset # Find the collection, merge properties if there are # common properties to merge. collection_to_merge = None if collection_id is not None and root is not None: collection_to_merge = root._resolved_objects.get_by_id( collection_id) else: collection_link = item.get_single_link('collection') if collection_link is not None: # If there's a relative collection link, and we have an href passed # in, we can resolve the collection from the link. If not, # we'll skip merging in collection properties. if collection_link.link_type == LinkType.RELATIVE and \ not collection_link.is_resolved(): if href is not None: collection_link = collection_link.clone() collection_link.target = make_absolute_href( collection_link.target, href) else: collection_link = None if collection_link is not None: collection_to_merge = collection_link.resolve_stac_object( root=root).target if item.collection_id is None: item.collection_id = collection_to_merge.id if collection_to_merge is not None: if collection_to_merge.properties is not None: item.properties = dict( ChainMap(item.properties, collection_to_merge.properties)) return item
def get_points_round(round_matches): matches_selected = round_matches.sort_values('gameweek') dicts = matches_selected.apply( lambda x: get_points_per_score(x['teamsData']), axis=1) return dict(ChainMap(*list(dicts)))
d.append('singh') print(d) d.appendleft('asmika') print(d) d.pop() #use popleft print(d) """chainmap is a dictionary like class for creating a single view of multiple mappings """ #Chainmap from collections import ChainMap a = {1: 'Hadoop', 2: 'BigData'} b = {3: 'Java', 4: 'Python'} temp = ChainMap(a, b) print(temp) """Counter is a dictionary subclass for counting hashable objects """ #Counter from collections import Counter a = [1, 2, 1, 3, 4, 2, 5, 1, 3, 7, 3, 1, 5, 7, 2] count = Counter(a) print(count) print(list(count.elements())) print(count.most_common()) sub = {2: 1, 6: 1} print(count.subtract(sub)) print(count.most_common())
print("\nSabor de la fruta:", frut.sabor) print("\n", "----------" * 7, "\n") # Importar de collections, ChainMap from collections import ChainMap # ChainMap: encapsula varios diccionarios en una unidad print("\nChainMap\n") # Ejercicio 10: juntar varios dicionarios print("\n- Ejercicio 10") # Creación de los diccionarios di1 = {"nombre": "César", "deporte": "natación"} di2 = {"juez": "Sonia", "edad": 32} di3 = {"premio": "trofeo", "lugar": "primer puesto"} # Mostrar los diccionarios print("\nDiccionario 1:", di1) print("\nDiccionario 2:", di2) print("\nDiccionario 3:", di3) # Uso de ChainMap chain = ChainMap(di1, di3, di2) # Mostrar el resultado print( "\nUsando ChainMap con los diccionarios anteriores, empezando por el 1, continuando con el 3 y terminando con el 2:\n", "\n", chain)
#!/usr/bin/env python # coding=utf-8 ''' 合并多个字典或映射 有多个字典或者映射,将它们从逻辑上合并为一个单一的映射后执行某些操作,比如查找值或者检查某些键是否存在。 ''' a = {'x': 1, 'z': 3} b = {'y': 2, 'z': 4} from collections import ChainMap c = ChainMap(a, b) print(c['x']) # Outputs 1 (from a) print(c['y']) # Outputs 2 (from b) print(c['z']) # Outputs 3 (from a) len(c) #3 list(c.keys()) #['x', 'y', 'z'] list(c.values()) #[1, 2, 3] c['z'] = 10 c['w'] = 40 del c['x'] print a #{'w': 40, 'z': 10} values = ChainMap() values['x'] = 1
mul6 = {'6*6': '24', '6*7': '2A', '6*8': '30', '6*9': '36', '6*A': '3C', '6*B': '42', '6*C': '48', '6*D': '4E', '6*E': '54', '6*F': '5A'} mul7 = {'7*7': '31', '7*8': '38', '7*9': '3F', '7*A': '46', '7*B': '4D', '7*C': '54', '7*D': '5B', '7*E': '62', '7*F': '69'} mul8 = {'8*8': '40', '8*9': '48', '8*A': '50', '8*B': '58', '8*C': '60', '8*D': '68', '8*E': '70', '8*F': '78'} mul9 = {'9*9': '51', '9*A': '5A', '9*B': '63', '9*C': '6C', '9*D': '75', '9*E': '7E', '9*F': '87'} mulA = {'A*A': '64', 'A*B': '6E', 'A*C': '78', 'A*D': '82', 'A*E': '8C', 'A*F': '96'} mulB = {'B*B': '79', 'B*C': '84', 'B*D': '8F', 'B*E': '9A', 'B*F': 'A5'} mulC = {'C*C': '90', 'C*D': '9C', 'C*E': 'A8', 'C*F': 'B4'} mulD = {'D*D': 'A9', 'D*E': 'B6', 'D*F': 'C3'} mulE = {'E*E': 'C4', 'E*F': 'D2'} mulF = {'F*F': 'E1'} # Объеденение всех словарей с помощью ChainMap operations = ChainMap(plus0, plus1, plus2, plus3, plus4, plus5, plus6, plus7, plus8, plus9, plusA, plusB, plusC, plusD, plusE, plusF, mul0, mul1, mul2, mul3, mul4, mul5, mul6, mul7, mul8, mul9, mulA, mulB, mulC, mulD, mulE, mulF) # Функция арифметической операции над двумя цифрами def hex_digit_calc(digit1: str, digit2: str, sign: str): try: return operations[digit1 + sign + digit2] except KeyError: try: return operations[digit2 + sign + digit1] except KeyError: return None # Функция сложения двух чисел
def _fd(self): return dict(ChainMap(*[getattr(i, '_fd', {}) for i in self._args_diff]))
import argparse import os from collections import ChainMap defaults = {"debug": False} parser = argparse.ArgumentParser() parser.add_argument('--debug') args = parser.parse_args() cli_args = {key: value for key, value in vars(args).items() if value} config = ChainMap(cli_args, os.environ, defaults) print(config.get('debug'))
d = defaultdict(lambda: 'N/A') print(d.get('key', 'test')) print(d['key']) # 按插入顺序排序的字典 print(OrderedDict({'a': 1, 'b': 2, 'd': 3, 'c': 4})) # 按定义的顺序查找 defaults = {'color': 'red', 'user': '******'} # 命令行参数解析 parser = argparse.ArgumentParser() parser.add_argument('-u', '--user') parser.add_argument('-c', '--color') namespace = parser.parse_args() command_line_args = {k: v for k, v in vars(namespace).items() if v} chain_map = ChainMap(command_line_args, os.environ, defaults) print(chain_map['user']) # 计数器 c = Counter() for ch in 'programming': c[ch] = c[ch] + 1 print(c) # ==================================================BASE64编码================================================= b = base64.encodebytes(b'yoo, cool') print(b) print(base64.decodebytes(b)) print(base64.urlsafe_b64encode(b'/user?name=guest'))
async def _process_command(self, command): '''Process a command from a client, and return the server response''' def get_db_entry(): chan = self.circuit.channels_sid[command.sid] db_entry = self.context[chan.name] return chan, db_entry if command is ca.DISCONNECTED: raise DisconnectedCircuit() elif isinstance(command, ca.VersionRequest): return [ca.VersionResponse(ca.DEFAULT_PROTOCOL_VERSION)] elif isinstance(command, ca.SearchRequest): pv_name = command.name try: self.context[pv_name] except KeyError: if command.reply == ca.DO_REPLY: return [ ca.NotFoundResponse( version=ca.DEFAULT_PROTOCOL_VERSION, cid=command.cid) ] else: return [ ca.SearchResponse(self.context.port, None, command.cid, ca.DEFAULT_PROTOCOL_VERSION) ] elif isinstance(command, ca.CreateChanRequest): try: db_entry = self.context[command.name] except KeyError: self.log.debug('Client requested invalid channel name: %s', command.name) return [ca.CreateChFailResponse(cid=command.cid)] access = db_entry.check_access(self.client_hostname, self.client_username) return [ ca.AccessRightsResponse(cid=command.cid, access_rights=access), ca.CreateChanResponse(data_type=db_entry.data_type, data_count=db_entry.max_length, cid=command.cid, sid=self.circuit.new_channel_id()), ] elif isinstance(command, ca.HostNameRequest): self.client_hostname = command.name elif isinstance(command, ca.ClientNameRequest): self.client_username = command.name elif isinstance(command, (ca.ReadNotifyRequest, ca.ReadRequest)): chan, db_entry = get_db_entry() try: data_type = command.data_type except ValueError: raise ca.RemoteProtocolError('Invalid data type') # If we are in the middle of processing a Write[Notify]Request, # allow a bit of time for that to (maybe) finish. Some requests # may take a long time, so give up rather quickly to avoid # introducing too much latency. await self.write_event.wait(timeout=WRITE_LOCK_TIMEOUT) metadata, data = await db_entry.auth_read( self.client_hostname, self.client_username, data_type, user_address=self.circuit.address) old_version = self.circuit.protocol_version < 13 if command.data_count > 0 or old_version: data = data[:command.data_count] # This is a pass-through if arr is None. data = apply_arr_filter(chan.channel_filter.arr, data) # If the timestamp feature is active swap the timestamp. # Information must copied because not all clients will have the # timestamp filter if chan.channel_filter.ts and command.data_type in ca.time_types: time_type = type(metadata) now = ca.TimeStamp.from_unix_timestamp(time.time()) metadata = time_type( **ChainMap({'stamp': now}, dict((field, getattr(metadata, field)) for field, _ in time_type._fields_))) notify = isinstance(command, ca.ReadNotifyRequest) data_count = db_entry.calculate_length(data) return [ chan.read(data=data, data_type=command.data_type, data_count=data_count, status=1, ioid=command.ioid, metadata=metadata, notify=notify) ] elif isinstance(command, (ca.WriteRequest, ca.WriteNotifyRequest)): chan, db_entry = get_db_entry() client_waiting = isinstance(command, ca.WriteNotifyRequest) async def handle_write(): '''Wait for an asynchronous caput to finish''' try: write_status = await db_entry.auth_write( self.client_hostname, self.client_username, command.data, command.data_type, command.metadata, user_address=self.circuit.address) except Exception as ex: self.log.exception('Invalid write request by %s (%s): %r', self.client_username, self.client_hostname, command) cid = self.circuit.channels_sid[command.sid].cid response_command = ca.ErrorResponse( command, cid, status=ca.CAStatus.ECA_PUTFAIL, error_message=('Python exception: {} {}' ''.format(type(ex).__name__, ex))) await self.send(response_command) else: if client_waiting: if write_status is None: # errors can be passed back by exceptions, and # returning none for write_status can just be # considered laziness write_status = True response_command = chan.write( ioid=command.ioid, status=write_status, data_count=db_entry.length) await self.send(response_command) finally: maybe_awaitable = self.write_event.set() # The curio backend makes this an awaitable thing. if maybe_awaitable is not None: await maybe_awaitable self.write_event.clear() await self._start_write_task(handle_write) elif isinstance(command, ca.EventAddRequest): chan, db_entry = get_db_entry() # TODO no support for deprecated low/high/to sub = Subscription(mask=command.mask, channel_filter=chan.channel_filter, channel=chan, circuit=self, data_type=command.data_type, data_count=command.data_count, subscriptionid=command.subscriptionid, db_entry=db_entry) sub_spec = SubscriptionSpec(db_entry=db_entry, data_type=command.data_type, mask=command.mask, channel_filter=chan.channel_filter) self.subscriptions[sub_spec].append(sub) self.context.subscriptions[sub_spec].append(sub) # If we are in the middle of processing a Write[Notify]Request, # allow a bit of time for that to (maybe) finish. Some requests # may take a long time, so give up rather quickly to avoid # introducing too much latency. if not self.write_event.is_set(): await self.write_event.wait(timeout=WRITE_LOCK_TIMEOUT) await db_entry.subscribe(self.context.subscription_queue, sub_spec, sub) elif isinstance(command, ca.EventCancelRequest): chan, db_entry = get_db_entry() removed = await self._cull_subscriptions( db_entry, lambda sub: sub.subscriptionid == command.subscriptionid) if removed: _, removed_sub = removed[0] data_count = removed_sub.data_count else: data_count = db_entry.length return [ chan.unsubscribe(command.subscriptionid, data_type=command.data_type, data_count=data_count) ] elif isinstance(command, ca.EventsOnRequest): # Immediately send most recent updates for all subscriptions. most_recent_updates = list(self.most_recent_updates.values()) self.most_recent_updates.clear() if most_recent_updates: await self.send(*most_recent_updates) maybe_awaitable = self.events_on.set() # The curio backend makes this an awaitable thing. if maybe_awaitable is not None: await maybe_awaitable self.circuit.log.info("Client at %s:%d has turned events on.", *self.circuit.address) elif isinstance(command, ca.EventsOffRequest): # The client has signaled that it does not think it will be able to # catch up to the backlog. Clear all updates queued to be sent... self.unexpired_updates.clear() # ...and tell the Context that any future updates from ChannelData # should not be added to this circuit's queue until further notice. self.events_on.clear() self.circuit.log.info("Client at %s:%d has turned events off.", *self.circuit.address) elif isinstance(command, ca.ClearChannelRequest): chan, db_entry = get_db_entry() await self._cull_subscriptions( db_entry, lambda sub: sub.channel == command.sid) return [chan.clear()] elif isinstance(command, ca.EchoRequest): return [ca.EchoResponse()]
def _flatten(list_of_dicts): list_of_dicts = [d for d in list_of_dicts if d != []] return dict(ChainMap(*list_of_dicts))
#!/usr/bin/python3 # -*- coding: utf-8 -*- # @time : 18-10-12 下午9:09 # @author : Feng_Hui # @email : [email protected] # how to use chainmap # https://docs.python.org/3/library/collections.html#collections.ChainMap from collections import ChainMap dict1 = {'a': 1, 'b': 2} dict2 = {'c': 3, 'd': 4} chain_map = ChainMap(dict1, dict2) print(chain_map, type(chain_map)) print(chain_map.get('a')) print(chain_map.maps) # mapping list dict2['e'] = 5 # 更新dict2 print(chain_map.maps) print(chain_map.parents) # 生成一个除第一个dict的其他所有dict的新ChainMap print(chain_map.new_child()) # 生成一个包含之前的ChainMap的新ChainMap,默认第一个dict为{} dict3 = {'f': 6} print(chain_map.new_child(dict3))
def __call__(self, *params): args = {name: val for name, val in zip(self.args, params)} env = ChainMap(args, self.env) return evaluate(self.body, env)
body = lispify(self.body) return f'(lambda ({args}) {body})' def begin(*args): if args: return args[-1] builtin = ChainMap({ '+': operator.add, '-': operator.sub, '*': operator.mul, '/': operator.truediv, '>': operator.gt, '<': operator.lt, '>=': operator.ge, '<=': operator.le, '=': operator.eq, '%': operator.mod, 'begin': begin, }) def evaluate(expr, env): # a if isinstance(expr, str): # variable return env[expr] # 2.3 if not isinstance(expr, list): # constant literal
def evaluate( self, model, distributed=False, half=False, trt_file=None, decoder=None, test_size=None, ): """ VOC average precision (AP) Evaluation. Iterate inference on the test dataset and the results are evaluated by COCO API. NOTE: This function will change training mode to False, please save states if needed. Args: model : model to evaluate. Returns: ap50_95 (float) : COCO style AP of IoU=50:95 ap50 (float) : VOC 2007 metric AP of IoU=50 summary (sr): summary info of evaluation. """ # TODO half to amp_test tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor model = model.eval() if half: model = model.half() ids = [] data_list = {} progress_bar = tqdm if is_main_process() else iter inference_time = 0 nms_time = 0 n_samples = len(self.dataloader) - 1 if trt_file is not None: from torch2trt import TRTModule model_trt = TRTModule() model_trt.load_state_dict(torch.load(trt_file)) x = torch.ones(1, 3, test_size[0], test_size[1]).cuda() model(x) model = model_trt for cur_iter, (imgs, _, info_imgs, ids) in enumerate( progress_bar(self.dataloader) ): with torch.no_grad(): imgs = imgs.type(tensor_type) # skip the the last iters since batchsize might be not enough for batch inference is_time_record = cur_iter < len(self.dataloader) - 1 if is_time_record: start = time.time() outputs = model(imgs) if decoder is not None: outputs = decoder(outputs, dtype=outputs.type()) if is_time_record: infer_end = time_synchronized() inference_time += infer_end - start outputs = postprocess( outputs, self.num_classes, self.confthre, self.nmsthre ) if is_time_record: nms_end = time_synchronized() nms_time += nms_end - infer_end data_list.update(self.convert_to_voc_format(outputs, info_imgs, ids)) statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples]) if distributed: data_list = gather(data_list, dst=0) data_list = ChainMap(*data_list) torch.distributed.reduce(statistics, dst=0) eval_results = self.evaluate_prediction(data_list, statistics) synchronize() return eval_results
full_pos_dist[k] = pd.DataFrame(full_pos_dist[k]) if debug_mode: import pdb pdb.set_trace() x = 1 return full_pos_dist all_distances, all_magnetometer = multiple_floors_train_predict( config, df, models_group_name, mode, holdout_df, test_floors, overwrite_models, test_type_mapping, only_public_test_preds, test_waypoint_times, debug_fn) if debug_fn is None: all_distances = dict(ChainMap(*[o for o in all_distances if o])) record_time = str(datetime.datetime.now())[:19] Path(preds_folder).mkdir(parents=True, exist_ok=True) file_ext = models_group_name + ' - ' + mode + save_ext + ( ' - full distances - ') + record_time + '.pickle' full_predictions_path = preds_folder / file_ext with open(full_predictions_path, 'wb') as handle: pickle.dump(all_distances, handle, protocol=pickle.HIGHEST_PROTOCOL) if mode == 'valid' and store_valid_magn_z: combined_magnetometer = pd.concat(all_magnetometer) combined_magnetometer.sort_values( ['site', 'floor', 'fn', 'center_sensor_time'], inplace=True) magn_ext = models_group_name + save_ext + ( ' - magnetometer - ') + record_time + '.csv'