def request(self, method: str, url: str, params: t.Dict = None, headers: t.Dict = None,
access_token: str = None, **options) -> t.Awaitable[t.Union[t.Dict, str]]:
"""Request OAuth2 resource."""
url = self._get_url(url)
headers = headers or {
'Accept': 'application/json',
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
}
access_token = access_token or self.access_token
if access_token:
headers.setdefault('Authorization', 'Bearer %s' % access_token)
return self._request(method, url, headers=headers, params=params, **options)
def floyd(nodes: typing.List, dist_dict: typing.Dict) -> typing.List:
"""
compute all-to-all shortest path by floyd algorithm
:param dist_dict: {(src, dst): distance}
:return: shortest_path: list of {"source": src, "target": dst,
"node_path": [{"name": name}],
"edge_path": [edge_0, edge_1, ..., edge_p]}, edge_i: {"source": src_i, "target": dst_i}
"""
MAX_LENGTH = float(len(nodes))
n_nodes = len(nodes)
path_dict = dict() # {(src, dst): shortest_path_next_hop_node_from_src}
for i in range(1, n_nodes):
for j in range(i):
path_dict[(min(nodes[j], nodes[i]),
max(nodes[j], nodes[i]))] = max(nodes[j], nodes[i])
for idx, bypass in enumerate(nodes):
print('{}/{}'.format(idx + 1, n_nodes))
for src in nodes:
for dst in nodes:
if src != dst and bypass != src and bypass != dst \
and (min(src, bypass), max(src, bypass)) in dist_dict \
and (min(dst, bypass), max(dst, bypass)) in dist_dict \
and dist_dict[(min(src, bypass), max(src, bypass))] + \
dist_dict[(min(bypass, dst), max(bypass, dst))] \
< dist_dict.setdefault((min(src, dst), max(src, dst)), MAX_LENGTH):
dist_dict[(min(src, dst), max(src, dst))] = \
dist_dict[(min(src, bypass), max(src, bypass))] + \
dist_dict[(min(bypass, dst), max(bypass, dst))]
path_dict[(min(src, dst),
max(src, dst))] = path_dict[(min(src, bypass),
max(src, bypass))]
shortest_path = list()
for i in range(1, n_nodes):
for j in range(i):
src, dst = min(nodes[j], nodes[i]), max(nodes[j], nodes[i])
tmp = {
"source": src,
"target": dst,
"node_path": list(({
"name": src
}, )),
"edge_path": list()
}
ptr = src
while ptr != dst:
pre = ptr
ptr = path_dict[(ptr, dst)]
tmp["node_path"].append({"name": ptr})
tmp["edge_path"].append({"source": pre, "target": ptr})
shortest_path.append(tmp)
return shortest_path
def dict_merge(source: typing.Dict, destination: typing.Dict) -> typing.Dict:
"""
run me with nosetests --with-doctest file.py
>>> a = { 'first' : { 'all_rows' : { 'pass' : 'dog', 'number' : '1' } } }
>>> b = { 'first' : { 'all_rows' : { 'fail' : 'cat', 'number' : '5' } } }
>>> dict_merge(b, a) == { 'first' : { 'all_rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }
True
"""
for key, value in source.items():
if isinstance(value, dict):
# get node or create one
node = destination.setdefault(key, {})
dict_merge(value, node)
else:
destination[key] = value
return destination
async def find_numbers(link: typing.AnyStr, numbers_catalog: typing.Dict):
logging.info('Working from "%s"' % link)
try:
numbers = numbers_catalog.setdefault(link, set())
async with aiohttp.ClientSession() as session:
response = await session.get(link, ssl=ssl.SSLContext())
html_content = await response.read()
text = _clear_html(html_content)
phone_numbers = [
_clean_phone(n[0]) for n in phone_format.findall(text) if n[0]
]
numbers.update(phone_numbers)
except Exception as e:
logger.exception('Ошибка при обработке "%s"' % link)
def map_storage(
fgraph: FunctionGraph,
order: typing.Iterable[Apply],
input_storage: typing.Optional[typing.List],
output_storage: typing.Optional[typing.List],
storage_map: typing.Dict = None,
) -> typing.Tuple[typing.List, typing.List, typing.Dict]:
"""Ensure there is storage (a length-1 list) for inputs, outputs, and interior nodes.
:param fgraph: The current fgraph. This function uses the inputs and outputs attributes.
:param order: an iterable over Apply instances (in program running order)
:param input_storage: None or existing input storage (see below)
:param output_storage: None or existing output storage (see below)
:rtype: 3-tuple
:returns: (list of storage for inputs, list of storage for outputs, and the `storage_map`)
Parameters
----------
fgraph
The current fgraph. This function uses the inputs and outputs
attributes.
order
An iterable over Apply instances (in program running order).
input_storage
None or existing input storage (see below).
output_storage
None or existing output storage (see below).
Returns
-------
3-tuple
List of storage for inputs, list of storage for outputs, and
the `storage_map`.
Extended summary
----------------
This function iterates over the nodes in `order` and ensures that for every
input and output `Variable`, there is a unique storage container. This is
returned as a dictionary Variable -> storage called the `storage_map`.
This function also returns `input_storage`, which is a list of storages
corresponding to fgraph.inputs.
This function also returns `output_storage`, which is a list of storages
corresponding to fgraph.outputs.
"""
# each Apply argument's data is stored in a list of length 1 (these lists act like pointers)
if storage_map is None:
storage_map = {}
# input_storage is a list of data-containers for the inputs.
if input_storage is None:
input_storage = [[None] for input in fgraph.inputs]
else:
assert len(fgraph.inputs) == len(input_storage)
# add input storage into storage_map
for r, storage in zip(fgraph.inputs, input_storage):
if r in storage_map:
assert storage_map[r] is storage, (
"Given input_storage conflicts "
"with storage in given storage_"
"map. Given input_storage: ",
storage,
"Storage in storage_ma" "p: ",
storage_map[r],
)
else:
storage_map[r] = storage
# for orphan in fgraph.orphans:
# if not isinstance(orphan, Constant):
# raise TypeError("Cannot link a graph with non-constant orphans.", orphan)
# storage_map[orphan] = [orphan.data]
# allocate output storage
if output_storage is not None:
assert len(fgraph.outputs) == len(output_storage)
for r, storage in zip(fgraph.outputs, output_storage):
if r in storage_map:
assert storage_map[r] is storage, (
"Given output_storage confl"
"icts with storage in given"
" storage_map. Given output"
"_storage: ",
storage,
"Sto" "rage in storage_map: ",
storage_map[r],
)
else:
storage_map[r] = storage
# allocate storage for intermediate computation
for node in order:
for r in node.inputs:
if r not in storage_map:
assert isinstance(r, Constant)
storage_map[r] = [r.data]
for r in node.outputs:
storage_map.setdefault(r, [None])
for r in fgraph.outputs:
if isinstance(r, Constant):
storage_map.setdefault(r, [r.data])
# extract output storage
if output_storage is None:
output_storage = [storage_map[r] for r in fgraph.outputs]
return input_storage, output_storage, storage_map
def parse_query(query_str: str, res: typing.Dict = None) -> typing.Dict:
"""
Basically takes a query string (like the ones in the examples of commands for the search__offers function) and
processes it into a dict of URL parameters to be sent to the server.
:param str query_str:
:param Dict res:
:return Dict:
"""
if res is None: res = {}
if type(query_str) == list:
query_str = " ".join(query_str)
query_str = query_str.strip()
opts = re.findall(
"([a-zA-Z0-9_]+)( *[=><!]+| +(?:[lg]te?|nin|neq|eq|not ?eq|not ?in|in) )?( *)(\[[^\]]+\]|[^ ]+)?( *)",
query_str)
# res = {}
op_names = {
">=": "gte",
">": "gt",
"gt": "gt",
"gte": "gte",
"<=": "lte",
"<": "lt",
"lt": "lt",
"lte": "lte",
"!=": "neq",
"==": "eq",
"=": "eq",
"eq": "eq",
"neq": "neq",
"noteq": "neq",
"not eq": "neq",
"notin": "notin",
"not in": "notin",
"nin": "notin",
"in": "in",
}
field_alias = {
"cuda_vers": "cuda_max_good",
"display_active": "gpu_display_active",
"reliability": "reliability2",
"dlperf_usd": "dlperf_per_dphtotal",
"dph": "dph_total",
"flops_usd": "flops_per_dphtotal",
}
field_multiplier = {
"cpu_ram": 1000,
"gpu_ram": 1000,
"duration": 1.0 / (24.0 * 60.0 * 60.0),
}
fields = {
"compute_cap",
"cpu_cores",
"cpu_cores_effective",
"cpu_ram",
"cuda_max_good",
"driver_version",
"disk_bw",
"disk_space",
"dlperf",
"dlperf_per_dphtotal",
"dph_total",
"duration",
"external",
"flops_per_dphtotal",
"gpu_display_active",
# "gpu_ram_free_min",
"gpu_mem_bw",
"gpu_name",
"gpu_ram",
"has_avx",
"host_id",
"id",
"inet_down",
"inet_down_cost",
"inet_up",
"inet_up_cost",
"min_bid",
"mobo_name",
"num_gpus",
"pci_gen",
"pcie_bw",
"reliability2",
"rentable",
"rented",
"storage_cost",
"total_flops",
"verified"
}
joined = "".join("".join(x) for x in opts)
if joined != query_str:
raise ValueError(
"Unconsumed text. Did you forget to quote your query? " +
repr(joined) + " != " + repr(query_str))
for field, op, _, value, _ in opts:
value = value.strip(",[]")
v = res.setdefault(field, {})
op = op.strip()
op_name = op_names.get(op)
if field in field_alias:
field = field_alias[field]
if not field in fields:
print(
"Warning: Unrecognized field: {}, see list of recognized fields."
.format(field),
file=sys.stderr)
if not op_name:
raise ValueError(
"Unknown operator. Did you forget to quote your query? " +
repr(op).strip("u"))
if op_name in ["in", "notin"]:
value = [x.strip() for x in value.split(",") if x.strip()]
if not value:
raise ValueError(
"Value cannot be blank. Did you forget to quote your query? " +
repr((field, op, value)))
if not field:
raise ValueError(
"Field cannot be blank. Did you forget to quote your query? " +
repr((field, op, value)))
if value in ["?", "*", "any"]:
if op_name != "eq":
raise ValueError("Wildcard only makes sense with equals.")
if field in v:
del v[field]
if field in res:
del res[field]
continue
if field in field_multiplier:
value = str(float(value) * field_multiplier[field])
v[op_name] = value
res[field] = v
return res
def register(self, registry: t.Dict):
registry = self.registry(registry)
for base in base_matrix(self.return_type):
registry.setdefault(base, set()).add(self)
def registry(cls, registry: t.Dict):
registry = registry.setdefault(cls, {})
return registry
