def request(self, method: str, url: str, params: t.Dict = None, headers: t.Dict = None, access_token: str = None, **options) -> t.Awaitable[t.Union[t.Dict, str]]: """Request OAuth2 resource.""" url = self._get_url(url) headers = headers or { 'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', } access_token = access_token or self.access_token if access_token: headers.setdefault('Authorization', 'Bearer %s' % access_token) return self._request(method, url, headers=headers, params=params, **options)
def floyd(nodes: typing.List, dist_dict: typing.Dict) -> typing.List: """ compute all-to-all shortest path by floyd algorithm :param dist_dict: {(src, dst): distance} :return: shortest_path: list of {"source": src, "target": dst, "node_path": [{"name": name}], "edge_path": [edge_0, edge_1, ..., edge_p]}, edge_i: {"source": src_i, "target": dst_i} """ MAX_LENGTH = float(len(nodes)) n_nodes = len(nodes) path_dict = dict() # {(src, dst): shortest_path_next_hop_node_from_src} for i in range(1, n_nodes): for j in range(i): path_dict[(min(nodes[j], nodes[i]), max(nodes[j], nodes[i]))] = max(nodes[j], nodes[i]) for idx, bypass in enumerate(nodes): print('{}/{}'.format(idx + 1, n_nodes)) for src in nodes: for dst in nodes: if src != dst and bypass != src and bypass != dst \ and (min(src, bypass), max(src, bypass)) in dist_dict \ and (min(dst, bypass), max(dst, bypass)) in dist_dict \ and dist_dict[(min(src, bypass), max(src, bypass))] + \ dist_dict[(min(bypass, dst), max(bypass, dst))] \ < dist_dict.setdefault((min(src, dst), max(src, dst)), MAX_LENGTH): dist_dict[(min(src, dst), max(src, dst))] = \ dist_dict[(min(src, bypass), max(src, bypass))] + \ dist_dict[(min(bypass, dst), max(bypass, dst))] path_dict[(min(src, dst), max(src, dst))] = path_dict[(min(src, bypass), max(src, bypass))] shortest_path = list() for i in range(1, n_nodes): for j in range(i): src, dst = min(nodes[j], nodes[i]), max(nodes[j], nodes[i]) tmp = { "source": src, "target": dst, "node_path": list(({ "name": src }, )), "edge_path": list() } ptr = src while ptr != dst: pre = ptr ptr = path_dict[(ptr, dst)] tmp["node_path"].append({"name": ptr}) tmp["edge_path"].append({"source": pre, "target": ptr}) shortest_path.append(tmp) return shortest_path
def dict_merge(source: typing.Dict, destination: typing.Dict) -> typing.Dict: """ run me with nosetests --with-doctest file.py >>> a = { 'first' : { 'all_rows' : { 'pass' : 'dog', 'number' : '1' } } } >>> b = { 'first' : { 'all_rows' : { 'fail' : 'cat', 'number' : '5' } } } >>> dict_merge(b, a) == { 'first' : { 'all_rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } True """ for key, value in source.items(): if isinstance(value, dict): # get node or create one node = destination.setdefault(key, {}) dict_merge(value, node) else: destination[key] = value return destination
async def find_numbers(link: typing.AnyStr, numbers_catalog: typing.Dict): logging.info('Working from "%s"' % link) try: numbers = numbers_catalog.setdefault(link, set()) async with aiohttp.ClientSession() as session: response = await session.get(link, ssl=ssl.SSLContext()) html_content = await response.read() text = _clear_html(html_content) phone_numbers = [ _clean_phone(n[0]) for n in phone_format.findall(text) if n[0] ] numbers.update(phone_numbers) except Exception as e: logger.exception('Ошибка при обработке "%s"' % link)
def map_storage( fgraph: FunctionGraph, order: typing.Iterable[Apply], input_storage: typing.Optional[typing.List], output_storage: typing.Optional[typing.List], storage_map: typing.Dict = None, ) -> typing.Tuple[typing.List, typing.List, typing.Dict]: """Ensure there is storage (a length-1 list) for inputs, outputs, and interior nodes. :param fgraph: The current fgraph. This function uses the inputs and outputs attributes. :param order: an iterable over Apply instances (in program running order) :param input_storage: None or existing input storage (see below) :param output_storage: None or existing output storage (see below) :rtype: 3-tuple :returns: (list of storage for inputs, list of storage for outputs, and the `storage_map`) Parameters ---------- fgraph The current fgraph. This function uses the inputs and outputs attributes. order An iterable over Apply instances (in program running order). input_storage None or existing input storage (see below). output_storage None or existing output storage (see below). Returns ------- 3-tuple List of storage for inputs, list of storage for outputs, and the `storage_map`. Extended summary ---------------- This function iterates over the nodes in `order` and ensures that for every input and output `Variable`, there is a unique storage container. This is returned as a dictionary Variable -> storage called the `storage_map`. This function also returns `input_storage`, which is a list of storages corresponding to fgraph.inputs. This function also returns `output_storage`, which is a list of storages corresponding to fgraph.outputs. """ # each Apply argument's data is stored in a list of length 1 (these lists act like pointers) if storage_map is None: storage_map = {} # input_storage is a list of data-containers for the inputs. if input_storage is None: input_storage = [[None] for input in fgraph.inputs] else: assert len(fgraph.inputs) == len(input_storage) # add input storage into storage_map for r, storage in zip(fgraph.inputs, input_storage): if r in storage_map: assert storage_map[r] is storage, ( "Given input_storage conflicts " "with storage in given storage_" "map. Given input_storage: ", storage, "Storage in storage_ma" "p: ", storage_map[r], ) else: storage_map[r] = storage # for orphan in fgraph.orphans: # if not isinstance(orphan, Constant): # raise TypeError("Cannot link a graph with non-constant orphans.", orphan) # storage_map[orphan] = [orphan.data] # allocate output storage if output_storage is not None: assert len(fgraph.outputs) == len(output_storage) for r, storage in zip(fgraph.outputs, output_storage): if r in storage_map: assert storage_map[r] is storage, ( "Given output_storage confl" "icts with storage in given" " storage_map. Given output" "_storage: ", storage, "Sto" "rage in storage_map: ", storage_map[r], ) else: storage_map[r] = storage # allocate storage for intermediate computation for node in order: for r in node.inputs: if r not in storage_map: assert isinstance(r, Constant) storage_map[r] = [r.data] for r in node.outputs: storage_map.setdefault(r, [None]) for r in fgraph.outputs: if isinstance(r, Constant): storage_map.setdefault(r, [r.data]) # extract output storage if output_storage is None: output_storage = [storage_map[r] for r in fgraph.outputs] return input_storage, output_storage, storage_map
def parse_query(query_str: str, res: typing.Dict = None) -> typing.Dict: """ Basically takes a query string (like the ones in the examples of commands for the search__offers function) and processes it into a dict of URL parameters to be sent to the server. :param str query_str: :param Dict res: :return Dict: """ if res is None: res = {} if type(query_str) == list: query_str = " ".join(query_str) query_str = query_str.strip() opts = re.findall( "([a-zA-Z0-9_]+)( *[=><!]+| +(?:[lg]te?|nin|neq|eq|not ?eq|not ?in|in) )?( *)(\[[^\]]+\]|[^ ]+)?( *)", query_str) # res = {} op_names = { ">=": "gte", ">": "gt", "gt": "gt", "gte": "gte", "<=": "lte", "<": "lt", "lt": "lt", "lte": "lte", "!=": "neq", "==": "eq", "=": "eq", "eq": "eq", "neq": "neq", "noteq": "neq", "not eq": "neq", "notin": "notin", "not in": "notin", "nin": "notin", "in": "in", } field_alias = { "cuda_vers": "cuda_max_good", "display_active": "gpu_display_active", "reliability": "reliability2", "dlperf_usd": "dlperf_per_dphtotal", "dph": "dph_total", "flops_usd": "flops_per_dphtotal", } field_multiplier = { "cpu_ram": 1000, "gpu_ram": 1000, "duration": 1.0 / (24.0 * 60.0 * 60.0), } fields = { "compute_cap", "cpu_cores", "cpu_cores_effective", "cpu_ram", "cuda_max_good", "driver_version", "disk_bw", "disk_space", "dlperf", "dlperf_per_dphtotal", "dph_total", "duration", "external", "flops_per_dphtotal", "gpu_display_active", # "gpu_ram_free_min", "gpu_mem_bw", "gpu_name", "gpu_ram", "has_avx", "host_id", "id", "inet_down", "inet_down_cost", "inet_up", "inet_up_cost", "min_bid", "mobo_name", "num_gpus", "pci_gen", "pcie_bw", "reliability2", "rentable", "rented", "storage_cost", "total_flops", "verified" } joined = "".join("".join(x) for x in opts) if joined != query_str: raise ValueError( "Unconsumed text. Did you forget to quote your query? " + repr(joined) + " != " + repr(query_str)) for field, op, _, value, _ in opts: value = value.strip(",[]") v = res.setdefault(field, {}) op = op.strip() op_name = op_names.get(op) if field in field_alias: field = field_alias[field] if not field in fields: print( "Warning: Unrecognized field: {}, see list of recognized fields." .format(field), file=sys.stderr) if not op_name: raise ValueError( "Unknown operator. Did you forget to quote your query? " + repr(op).strip("u")) if op_name in ["in", "notin"]: value = [x.strip() for x in value.split(",") if x.strip()] if not value: raise ValueError( "Value cannot be blank. Did you forget to quote your query? " + repr((field, op, value))) if not field: raise ValueError( "Field cannot be blank. Did you forget to quote your query? " + repr((field, op, value))) if value in ["?", "*", "any"]: if op_name != "eq": raise ValueError("Wildcard only makes sense with equals.") if field in v: del v[field] if field in res: del res[field] continue if field in field_multiplier: value = str(float(value) * field_multiplier[field]) v[op_name] = value res[field] = v return res
def register(self, registry: t.Dict): registry = self.registry(registry) for base in base_matrix(self.return_type): registry.setdefault(base, set()).add(self)
def registry(cls, registry: t.Dict): registry = registry.setdefault(cls, {}) return registry