def threeSum__brute_force__v2(self,
num_list: List[int]) -> List[List[int]]:
"""
Solution to "three sum" that:
- loops through all triplets
- stores results in a hash set
"""
target = 0
match_hash = DefaultDict(int)
#===========================================================
# Iterate through every combination...
n = len(num_list)
for i1 in range(0, n - 2):
for i2 in range(i1 + 1, n - 1):
for i3 in range(i2 + 1, n):
num1 = num_list[i1]
num2 = num_list[i2]
num3 = num_list[i3]
trial = tuple(sorted((num1, num2, num3)))
if sum(trial) == target:
match_hash[trial] += 1
return tuple(match_hash.keys())
def rand(self):
""""""
rand_dict = DefaultDict(dict)
for hp in self:
rand_dict[hp.section][hp.option] = hp.rand()
return rand_dict
async def collect_types(session, roots):
types = DefaultDict(set)
tsets = set()
done = 0
last = None
async def visit(title, comm, limit, *,
task_status=TASK_STATUS_IGNORED):
nonlocal done, last
async with limit:
task_status.started()
with exceptions(comm.url):
ts = [(t, n) for t, (c, n)
in (await comm.types(session)).items()
if c.casefold() == 'результаты выборов']
for t, n in ts: types[t].add(n)
tsets.add(tuple(t for t, n in ts))
sing = [await comm.single(session, t)
for t, n in ts if not n.startswith('Сводн')]
aggr = [await comm.aggregate(session, t)
for t, n in ts if n.startswith('Сводн')]
pprint((comm.url, await comm.path(session), await comm.results(session)), max_width=160)
skeys = [frozenset(v.name for v in res.votes)
for res in sing if res]
akeys = [frozenset(v.name for v in ress.popitem()[1].votes)
for ress in aggr if ress]
assert list(sorted(set(skeys), key=str)) == list(sorted(skeys, key=str))
assert list(sorted(set(akeys), key=str)) == list(sorted(akeys, key=str))
assert set(akeys) <= set(skeys)
last = ('/'.join(c if c is not None else '!'
for c in await comm.path(session)) +
': ' + title)
done += 1
async def traverse(nursery, root, limit):
title = str(await root.date(session)) + ' ' + root.title
with exceptions(root.url):
async with root.walk(session, 2) as children:
async for comm in children:
await nursery.start(visit, title, comm, limit)
try:
limit = CapacityLimiter(100)
async with open_nursery() as nursery:
for root in roots:
nursery.start_soon(traverse, nursery, root, limit)
report(done, len(nursery.child_tasks), last)
await sleep(0)
while nursery.child_tasks:
report(done, len(nursery.child_tasks), last)
await sleep(0)
finally:
clear()
pprint(dict(types), max_width=w)
pprint(list(map(list, tsets)), max_width=w)
def _counter__python_defaultdict(self, iterable: Iterable) -> DefaultDict:
# Create a dict counting the elements of an iterable.
counter = DefaultDict(int)
for item in iterable:
counter[item] += 1
return counter
def improve_rank(self, dims=None):
""""""
hps = [
hp for i, hp in enumerate(self)
if (not hp.fixed) and ((dims is None) or (i in dims))
]
d = len(hps)
discretes = [
hp for hp in hps
if isinstance(hp, (BoolHyperparam, StringHyperparam))
]
discrete_values = {}
for discrete in discretes:
for i, bound in enumerate(discrete.bounds):
discrete_values[(discrete.name,
bound)] = np.equal(discrete.values,
i).astype(int)
discrete_list = []
for i, discrete1 in enumerate(discretes):
for bound1 in discrete1.bounds:
values1 = discrete_values[(discrete1.name, bound1)]
for j, discrete2 in enumerate(discretes):
if i != j:
for bound2 in discrete2.bounds:
values2 = discrete_values[(discrete2.name, bound2)]
discrete_list.append(
(np.dot(values1, values2), discrete1.name,
bound1, discrete2.name, bound2))
elif i == j:
discrete_list.append(
(np.dot(values1, values1), discrete1.name, bound1,
discrete1.name, bound1))
discrete_list.sort()
rand_dict = DefaultDict(dict)
hps_to_remove = set()
for count, name1, bound1, name2, bound2 in discrete_list:
section1, option1 = name1.split(':')
section2, option2 = name2.split(':')
if not (section1 in rand_dict and option1 in rand_dict[section1]) or\
(section1 in rand_dict and option1 in rand_dict[section1] and bound1 == rand_dict[section1][option1]):
if not (section2 in rand_dict and option2 in rand_dict[section2]) or\
(section2 in rand_dict and option2 in rand_dict[section2] and bound2 == rand_dict[section2][option2]):
if count <= d:
rand_dict[section1][option1] = bound1
rand_dict[section2][option2] = bound2
hps_to_remove.add((section1, option1))
hps_to_remove.add((section2, option2))
if dims is not None or hps_to_remove:
dims = [
i for i, hp in enumerate(self)
if ((i in dims) and ((hp.section,
hp.option) not in hps_to_remove))
]
return rand_dict, np.array(dims)
def threeSum__hash__v1(self, num_list: List[int]) -> List[List[int]]:
"""
Solution to "three sum" that
- uses a hash map from numbers to indices
- stores results in a hash set
"""
target = 0
match_hash = DefaultDict(int)
#===========================================================
# Hash numbers to indices...
num_hash = DefaultDict(set)
for (i, num) in enumerate(num_list):
num_hash[num].add(i)
#===========================================================
# Iterate through combinations...
n = len(num_list)
for i1 in range(0, n - 2):
for i2 in range(i1 + 1, n - 1):
num1 = num_list[i1]
num2 = num_list[i2]
num3 = target - num1 - num2
if num3 in num_hash:
# Find whether i3 is available. We cannot duplicate i1 or i2.
i3_set = set.difference(num_hash[num3], set((i1, i2)))
if i3_set:
trial = tuple(sorted((num1, num2, num3)))
match_hash[trial] += 1
return tuple(match_hash.keys())
def sort_keys(output):
"""Parse gpg lines."""
valid = DefaultDict(list)
invalid = DefaultDict(list)
unknown = DefaultDict(list)
today = date.today()
for line in output.splitlines():
# process lines that start with "pub" or "sub"
if not True in map(line.startswith, ("pub", "sub")):
continue
key, status, created, expires = parse(line)
if status in TRUSTWORTHY_STATES:
valid[expires].append(key)
else:
invalid[expires].append(key)
return valid, invalid, unknown
def __init__(self, *args, placeholder_shape=[None,None,None], **kwargs):
""""""
super(FeatureVocab, self).__init__(*args, placeholder_shape=placeholder_shape, **kwargs)
self._counts = DefaultDict(Counter)
self._str2idx = DefaultDict(dict)
self._idx2str = DefaultDict(dict)
self.PAD_STR = self.UNK_STR = self.pad_str
self.PAD_IDX = self.UNK_IDX = 0
if self.keyed:
self.ROOT_STR = 'Yes'
self.ROOT_IDX = 1
self._feats = ['Root']
self._feat_set = {'Root'}
self['Root', self.PAD_STR] = self.PAD_IDX
self['Root', self.ROOT_STR] = self.ROOT_IDX
else:
self.ROOT_STR = self.pad_str
self.ROOT_IDX = 0
self._feats = list()
self._feat_set = set()
def finder(paths, queries):
# map bases to paths, allowing duplicates
base_to_paths_map = DefaultDict(list)
for p in paths:
b = os.path.basename(p)
base_to_paths_map[b].append(p)
# find paths matching queries
matches = []
for q in queries:
if q in base_to_paths_map:
matches.extend(base_to_paths_map[q])
return matches
def get_indices_of_item_weights(weights, length, limit):
# map weights to indices, allowing duplicates:
weight_to_indices_map = DefaultDict(list)
for (i, w) in enumerate(weights):
weight_to_indices_map[w].append(i)
# set default `matches`
matches = None
# find indices of weights which sum to limit:
for (w1, i1s) in weight_to_indices_map.items():
# the indices of weight 1
i1s = weight_to_indices_map[w1]
# the next necessary weight
w2 = limit - w1
# the indices of weight 2, maybe...
i2s = weight_to_indices_map[w2] if w2 in weight_to_indices_map else None
# if there could be any, try to get matching indices
if i1s and i2s:
# if weights are different, use first index of both
if w1 != w2:
matches = (i1s[0], i2s[0])
break
# if weights are same, there need to be enough indices
# i1s == i2s, in this case
elif len(i1s) > 1:
matches = tuple(i1s[0:2])
break
return matches
def twoSum(self, nums: List[int], target: int) -> List[int]:
# make dict of nums, with possible repeats
nums_dict = DefaultDict(list)
for (i, num) in enumerate(nums):
nums_dict[num].append(i)
# search nums_dict for target
for (i_1, num_1) in enumerate(nums):
num_2 = target - num_1
if num_2 in nums_dict:
i_2s = nums_dict[num_2]
i_2 = None
# handle possible repeats
if i_1 in i_2s:
if len(i_2s) > 1:
i_2 = tuple(i for i in i_2s if i != i_1)[0]
else:
i_2 = i_2s[0]
# return if match found
if i_2 is not None:
return [i_1, i_2]
return []
def threeSum__brute_force__v3(self,
num_list: List[int]) -> List[List[int]]:
"""
Solution to "three sum" that
- loops through all triplets
- uses hash sets of num1, num2, num3 to avoid duplicates
- stores results in a hash set
"""
target = 0
match_hash = DefaultDict(int)
#===========================================================
# Iterate through every combination...
n = len(num_list)
# Track duplicates of num1.
visited_num1_hash = set()
for i1 in range(0, n - 2):
num1 = num_list[i1]
# Skip duplicate of num1, else record it.
if num1 in visited_num1_hash:
continue
else:
visited_num1_hash.add(num1)
# Track duplicates of num2 (reseting for every num1).
visited_num2_hash = set()
for i2 in range(i1 + 1, n - 1):
num2 = num_list[i2]
# Skip duplicate of num2, else record it.
if num2 in visited_num2_hash:
continue
else:
visited_num2_hash.add(num2)
# Track duplicates of num3 (reseting for every num2 and num1).
visited_num3_hash = set()
for i3 in range(i2 + 1, n):
num3 = num_list[i3]
# Skip duplicate of num3, else record it.
if num3 in visited_num3_hash:
continue
else:
visited_num3_hash.add(num3)
trial = tuple(sorted((num1, num2, num3)))
if sum(trial) == target:
match_hash[trial] += 1
return tuple(match_hash.keys())
def threeSum__hash__v2(self, num_list: List[int]) -> List[List[int]]:
"""
Solution to "three sum" that
- uses a hash map from numbers to indices
- uses hash sets of num1, num2 to avoid duplicates
- stores results in a hash set
"""
target = 0
match_hash = DefaultDict(int)
#===========================================================
# Hash numbers to indices...
num_hash = DefaultDict(set)
for (i, num) in enumerate(num_list):
num_hash[num].add(i)
#===========================================================
# Iterate through combinations...
n = len(num_list)
# Track duplicates of num1.
visited_num1_hash = set()
for i1 in range(0, n - 2):
num1 = num_list[i1]
# Skip duplicate of num1, else record it.
if num1 in visited_num1_hash:
continue
else:
visited_num1_hash.add(num1)
# Track duplicates of num2 (reseting for every num1).
visited_num2_hash = set()
for i2 in range(i1 + 1, n - 1):
num2 = num_list[i2]
# Skip duplicate of num2, else record it.
if num2 in visited_num2_hash:
continue
else:
visited_num2_hash.add(num2)
num3 = target - num1 - num2
if num3 in num_hash:
# Find whether i3 is available. We cannot duplicate i1 or i2.
i3_set = set.difference(num_hash[num3], set((i1, i2)))
if i3_set:
trial = tuple(sorted((num1, num2, num3)))
match_hash[trial] += 1
return tuple(match_hash.keys())
def rand(self, dims=None):
""""""
if dims is not None:
dims = set(dims)
#-----------------------------------------------------------
def compute_weights(scores):
""" computes softmax(log(len(scores)) * scores) """
scores = scores - np.max(scores)
exp_scores = len(scores)**scores
weights = exp_scores / exp_scores.sum()
return weights
#-----------------------------------------------------------
finite_scores = np.isfinite(self.scores)
hps = [
hp for i, hp in enumerate(self)
if (not hp.fixed) and ((dims is None) or (i in dims))
]
d = 0
for hp in hps:
if isinstance(hp, StringHyperparam):
d += len(hp.bounds) - 1
else:
d += 1
# Compute the weighted means
mean = np.zeros(d)
i = 0
for hp in hps:
locs = np.where(np.isfinite(hp.values) * finite_scores)[0]
if isinstance(hp, NumericHyperparam):
all_values = hp.values[locs][None, :]
else:
all_values = np.zeros([len(hp.bounds) - 1, len(locs)])
for j in six.moves.range(len(hp.bounds) - 1):
all_values[j, np.where(hp.values[locs] == j)] = 1
weights = compute_weights(self.scores[locs])
for j, values in enumerate(all_values):
mean[i + j] = np.dot(values, weights)
i += j + 1
# Compute the weighted covariance
cov = np.zeros([d, d])
i1 = 0
for hp1 in hps:
i2 = 0
for hp2 in hps:
locs = np.where(
np.isfinite(hp1.values) * np.isfinite(hp2.values) *
finite_scores)[0]
if isinstance(hp1, NumericHyperparam):
all_values1 = hp1.values[locs][None, :]
else:
all_values1 = np.zeros([len(hp1.bounds) - 1, len(locs)])
for j in six.moves.range(len(hp1.bounds) - 1):
all_values1[j, np.where(hp1.values[locs] == j)] = 1
if isinstance(hp2, NumericHyperparam):
all_values2 = hp2.values[locs][None, :]
else:
all_values2 = np.zeros([len(hp2.bounds) - 1, len(locs)])
for j in six.moves.range(len(hp2.bounds) - 1):
all_values2[j, np.where(hp2.values[locs] == j)] = 1
for j1, values1 in enumerate(all_values1):
values1 = hp1.values[locs] - mean[i1 + j1]
bool_values1 = hp1.values[locs].astype(bool)
for j2, values2 in enumerate(all_values2):
if j2 <= j1:
values2 = hp2.values[locs] - mean[i2 + j2]
bool_values2 = hp2.values[locs].astype(bool)
weights = compute_weights(self.scores[locs])
value = np.dot(weights * values1, values2) / (
1 - np.dot(weights, weights))
cov[i1 + j1, i2 + j2] = cov[i2 + j2,
i1 + j1] = value
i2 += j2 + 1
i1 += j1 + 1
cov += .05**2 * np.eye(len(cov))
eigenvals, eigenvecs = la.eig(cov)
eigenvals = np.abs(eigenvals)
cov = (np.abs(eigenvals[:, None]) * eigenvecs).dot(eigenvecs.T)
if la.matrix_rank(cov) < len(cov):
print('WARNING: indefinite covariance matrix')
return {}
rand_dict = DefaultDict(dict)
vals = np.clip(np.random.multivariate_normal(mean, cov), 0, 1)
i = 0
for hp in hps:
if isinstance(hp, NumericHyperparam):
rand_dict[hp.section][hp.option] = hp.denormalize(vals[i])
i += 1
else:
rand_dict[hp.section][hp.option] = hp.denormalize(
vals[i:i + len(hp.bounds) - 1])
i += len(hp.bounds) - 1
return rand_dict
def rand(self, dims=None):
""""""
if dims is not None:
dims = set(dims)
hps = [
hp for i, hp in enumerate(self)
if (not hp.fixed) and ((dims is None) or (i in dims))
]
mat = [np.ones([len(self.scores), 1])] + [hp.as_matrix() for hp in hps]
mat = np.concatenate(mat, axis=1)
d = mat.shape[1] - 1
interactmat = []
for i, vec1 in enumerate(mat.T):
for j, vec2 in enumerate(mat.T):
if i <= j:
interactmat.append((vec1 * vec2)[:, None])
X = np.concatenate(interactmat, axis=1)
n, d2 = X.shape
I = np.eye(d2)
I[0, 0] = 0
XTXinv = la.inv(X.T.dot(X) + .05 * I)
# TODO maybe: L1 regularization on the interactions
mean = XTXinv.dot(X.T).dot(self.scores)
H = X.dot(XTXinv).dot(X.T)
epsilon_hat = self.scores - H.dot(self.scores)
dof = np.trace(np.eye(n) - H)
s_squared = epsilon_hat.dot(epsilon_hat) / dof
cov = s_squared * XTXinv
eigenvals, eigenvecs = la.eig(cov)
eigenvals = np.diag(np.abs(eigenvals))
eigenvecs = np.real(eigenvecs)
cov = eigenvecs.dot(eigenvals).dot(eigenvecs.T)
cov += .05**2 * np.eye(len(cov))
if la.matrix_rank(cov) < len(cov):
print('WARNING: indefinite covariance matrix')
return {}
rand_dict = DefaultDict(dict)
vals = np.random.multivariate_normal(mean, cov)
bias = vals[0]
lins = vals[1:d + 1]
bilins = np.zeros([d, d])
bilins[np.tril_indices(d)] = vals[d + 1:]
bilins = .5 * bilins + .5 * bilins.T
eigenvals, eigenvecs = la.eig(bilins)
eigenvals = -np.diag(np.abs(eigenvals))
eigenvecs = np.real(eigenvecs)
bilins = eigenvecs.dot(eigenvals).dot(eigenvecs.T)
if la.matrix_rank(bilins) < len(bilins):
print('WARNING: indefinite interaction matrix')
return {}
rand_dict = DefaultDict(dict)
vals = np.clip(.5 * la.inv(bilins).dot(lins), 0, 1)
i = 0
for hp in hps:
if isinstance(hp, NumericHyperparam):
rand_dict[hp.section][hp.option] = hp.denormalize(vals[i])
i += 1
else:
rand_dict[hp.section][hp.option] = hp.denormalize(
vals[i:i + len(hp.bounds) - 1])
i += len(hp.bounds) - 1
return rand_dict
def __init__(self,
listeners=None,
senders=None,
hosting=None,
consuming=None,
monitors=None,
my_key_id=0,
reply_service=None,
save_dir=".",
save_services=True,
gpg=None,
force_sender=None,
*args,
**kwargs):
"""Create a Santiago with the specified parameters.
listeners and senders are both connector-specific dictionaries containing
relevant settings per connector:
{ "http": { "port": 80 } }
*hosting* and *consuming* are service dictionaries. *hosting* contains
services you host, while *consuming* lists services you use, as a
client.
hosting: { "someKey": { "someService": ( "http://a.list",
"http://of.locations" )}}
consuming: { "someKey": { "someService": ( "http://a.list",
"http://of.locations" )}}
Messages are delivered by defining both the source and destination
("from" and "to", respectively). Separating this from the hosting and
consuming allows users to safely proxy requests for one another, if some
hosts are unreachable from some points.
:my_key_id: my PGP key ID.
:reply_service: Messages between clients contain lists of keys, one of
which is the "reply to" location. This parameter names the key to
check for reply locations in messages. This is usually
"freedombuddy".
:save_dir: The directory to save service data to, for storage between
sessions.
:save_services: Whether to save service data between sessions at all.
Technically, it's "whether service data is overwritten at the end of
the session", but that's mostly semantics.
"""
super(Santiago, self).__init__(*args, **kwargs)
self.live = 1
self.requests = DefaultDict(set)
self.my_key_id = my_key_id
self.gpg = gpg or gnupg.GPG(use_agent=True)
self.connectors = set()
self.reply_service = reply_service or Santiago.SERVICE_NAME
self.save_services = save_services
self.force_sender = force_sender #if force_sender in senders else None
self.listeners = self.create_connectors(listeners, "Listener")
self.senders = self.create_connectors(senders, "Sender")
self.monitors = self.create_connectors(monitors, "Monitor")
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
self.shelf = shelve.open(
save_dir.rstrip(os.sep) + os.sep + str(self.my_key_id) + ".dat")
self.hosting = hosting if hosting else self.load_data("hosting")
self.consuming = consuming if consuming else self.load_data(
"consuming")