def cons_sanity(self):
'''Return sanity-check constraints.
The returned list of constraints will require that for every three
categories A, B, and C, AB[i,j] & BC[j,k] -> AC[i,k].
For example, if your categories are first name, last name, and hair
color, this will emit constraints ensuring that if the first name
'Alice' goes with the last name 'Svetlana' and the last name 'Svetlana'
goes with the hair color 'Teal', then the first name 'Alice' must go
with the hair color 'Teal'.
'''
for f1, f2, f3 in combos(self.categories, 3):
a2b = self.vgrids[f1][f2]
b2c = self.vgrids[f2][f3]
a2c = self.vgrids[f1][f3]
for i in range(self.num_items):
for j in range(self.num_items):
for k in range(self.num_items):
yield z3.Implies(
z3.And(a2b[i, j], b2c[j, k]), a2c[i, k])
for cat in self.xcats:
for f1, f2 in combos(self.categories, 2):
a2b = self.vgrids[f1][f2]
cat1 = self.vcats[cat][f1]
cat2 = self.vcats[cat][f2]
for i in range(self.num_items):
for j in range(self.num_items):
yield (a2b[i, j] == False) | (cat1[i] == cat2[j])
def grid(self) -> CatGrid:
if self._grid is None:
altrows = [dict(r) for r in self.rows]
cats = list(altrows[0])
# Deduplicate non-unique rows
for cat in cats:
vals = set()
for row in altrows:
while row[cat] in vals:
row[cat] = '_{}'.format(row[cat])
vals.add(row[cat])
# Add xcat rows to grid, using only values that were actually used
catmap = {cat: [i.val for i in v]
for (cat, v) in self.catprob.catmap.items()}
for cat in set(cats) - set(catmap):
catmap[cat] = [row[cat] for row in altrows]
self._grid = CatGrid(catmap)
for f1, f2 in combos(cats, 2):
self._grid.grids[f1][f2][:, :] = False
for row in altrows:
for f1, f2 in combos(cats, 2):
ia = self._grid.get_info(row[f1], cat=f1)
ib = self._grid.get_info(row[f2], cat=f2)
self._grid.grids[f1][f2][ia.idx, ib.idx] = True
return self._grid
def geometric_voting_rel(boxes, wmax, wmin, rtol, intersect_mode, verb=False):
global centers, w_max, w_min, radii, r_rel_tol, vote_db, relative_search, verbose
relative_search = True
vote_db = {}
r_rel_tol = rtol
w_max = wmax
w_min = wmin
n = boxes.shape[0]
radii = np.zeros(n)
center_points = np.zeros((n, 2))
# approximate all radii in pixels
for i, box in enumerate(boxes):
radii[i] = util.calculate_radius(box)
center_points[i] = util.calculate_center(box)
centers = center_points
# get sorting vector
sort = radii.argsort()[::-1]
# sort dat shit
radii = radii[sort]
tuple_generator = combos(np.arange(n), 2)
for count, (ci, cj) in enumerate(tuple_generator):
# get pair-database indices of pair candidates
if intersect_mode:
ci_candidates, cj_candidates = search_pair(ci, cj, True)
add_votes_to_db(vote_db, ci, ci_candidates)
add_votes_to_db(vote_db, cj, cj_candidates)
else:
ci_d, ci_r, cj_d, cj_r = search_pair(ci, cj, False)
add_votes_to_db(vote_db, ci, ci_d)
add_votes_to_db(vote_db, ci, ci_r)
add_votes_to_db(vote_db, cj, cj_d)
add_votes_to_db(vote_db, cj, cj_r)
return do_final_validation(), vote_db
def test_cache_ops():
fsm = FileStorageManager()
ops = [sub, add]
tseries = []
size_tot = 0
ts = list(range(0, randint(0, 10000)))
for i in range(5):
vs = [randint(0, 1000000) for t in ts]
ats = ArrayTimeSeries(ts, vs)
fsm.store(i, ats)
tseries.append(ats)
size_tot += getsizeof(ats)
fsm = FileStorageManager(max_cache_size=10 * size_tot)
for o in ops:
for i, j in combos(range(5), 2):
t1 = SMTimeSeries.from_db(i, fsm)
t2 = SMTimeSeries.from_db(j, fsm)
t3 = o(t1, t2)
assert o(t1, t2) == fsm._cache[t3._ident]
t = -SMTimeSeries.from_db(0)
assert -tseries[0] == fsm._cache[t._ident]
def exclusion_by_intersection_rel(boxes, wmax, wmin, rtol, verb=False):
global centers, w_max, w_min, radii, r_rel_tol, intersect_db, relative_search, verbose
relative_search = True
verbose = verb
intersect_db = {}
r_rel_tol = rtol
w_max = wmax
w_min = wmin
n = boxes.shape[0]
radii = np.zeros(n)
center_points = np.zeros((n, 2))
# approximate all radii in pixels
for i, box in enumerate(boxes):
radii[i] = util.calculate_radius(box)
center_points[i] = util.calculate_center(box)
centers = center_points
# get sorting vector
sort = radii.argsort()[::-1]
# sort dat shit
radii = radii[sort]
tuple_generator = combos(np.arange(n), 2)
for ci, cj in tuple_generator:
# get pair-database indices of pair candidates
ci_candidates, cj_candidates = search_pair(ci, cj, True)
add_ids_to_db(intersect_db, ci, ci_candidates)
add_ids_to_db(intersect_db, cj, cj_candidates)
return do_final_validation()
def reduceByCheckCount(check_count, top_layer, bot_layer, top_reachable_idxs):
fraction_reachable_top = (len(top_reachable_idxs) - 0.1) / len(top_layer)
lowest = 10**6
result = []
for top_subset_idxs in combos(top_reachable_idxs, check_count):
top_remaining = [
top_layer[idx] for idx in top_reachable_idxs
if idx not in top_subset_idxs
]
reachable_bot = [
v for v in bot_layer if any(
manhatDist(u, v) == 1 for u in top_remaining)
]
# print(top_subset_idxs, reachable_bot)
fraction_reachable_bot = len(reachable_bot) - 0.1
fraction_reachable_bot /= len(bot_layer)
isDecreasing = (fraction_reachable_bot < fraction_reachable_top)
isMonotonic = (fraction_reachable_bot <= fraction_reachable_top)
evenCond = EVEN and isDecreasing
oddCond = not EVEN and isMonotonic
if evenCond or oddCond:
if fraction_reachable_bot < lowest:
result = []
lowest = fraction_reachable_bot
bot_reachable_idxs = [bot_layer.index(v) for v in reachable_bot]
result.append((check_count, sorted(bot_reachable_idxs)))
return result
def write_all_compare_results(dds, treats, r, shrink=False, quiet=True, remove_pvals=False):
writer = pd.ExcelWriter(
'../Data/significantly_different_expressions_shrink:{0}_onlyDiff:{1}.xlsx'.format(shrink,
remove_pvals))
for i, j in combos(treats, 2):
if not quiet:
print("Working on {0} | {1}".format(i, j))
if shrink:
r_res = r.lfcShrink(dds, contrast=ro.StrVector(
["treatmentID", i, j]))
else:
r_res = r.results(dds, contrast=ro.StrVector(
["treatmentID", i, j]))
r_res = r['as.data.frame'](r_res)
df = R_to_pandas(r_res, r)
df = df.sort_values(by=['padj'])
if remove_pvals:
df = df[df['padj'] < 0.01]
conv = {k: str(k) for k in list(df.columns)}
df = df.rename(index=str, columns=conv)
df.to_excel(writer, sheet_name="{0} | {1}".format(i, j))
writer.save()
writer.close()
def sum_not_ab():
ab = (n for n in range(12, 28123) if abun(n))
sum_ab = set(a + b for a, b in combos(ab, 2))
not_ab = (x for x in range(28123) if x not in sum_ab)
return (sum(not_ab))
def calc_similarities(R, sim_func=pearson):
"""Compute the pairwise similarities between each user based on their
ratings.
Parameters
----------
R : pandas.DataFrame
Movie ratings for each user.
sim_func : function, optional
Function of the form ``sim_func(R, u, v)`` that computes the similarity
between users ``u`` and ``v`` from the ratings dataframe ``R``.
Returns
-------
S : pandas.Dataframe
Dataframe containing the pairwise similarities between users.
"""
# Initialise an empty dataframe of similarities.
S = pandas.DataFrame(index=R.index, columns=R.index)
S.fillna(0.0, inplace=True)
# Assign the pairwise similarities.
for (u, _), (v, _) in combos(R.iterrows(), 2):
S.ix[u, v] = S.ix[v, u] = sim_func(R, u, v)
return S
def addHDMRLevel(self,varnames,lvl):
todo={}
nameset = combos(varnames,lvl)
for entry in nameset:
todo[entry]=[]
for e,ent in enumerate(entry):
todo[entry].append(self.varDict[ent])
return todo
def make_some_features(numbers, clip):
features = set()
combinations = (combo for combo in combos(numbers, 6))
for i, comb in enumerate(combinations):
if i % clip == 0: # Keeping size reasonable
for perm in perms(comb):
features.add(perm)
return features
def addHDMRLevel(self, varnames, lvl):
todo = {}
nameset = combos(varnames, lvl)
for entry in nameset:
todo[entry] = []
for e, ent in enumerate(entry):
todo[entry].append(self.varDict[ent])
return todo
def get_combos(pos_avail, num, limit_cost):
all = list(combos(pos_avail, num))
bos = [ComboPlayerSet(x) for x in all]
get = []
for x in bos:
if x.total_cost / num < limit_cost:
get.append(x)
return get
def get_combos(pos_avail, num, limit_cost):
all = list(combos(pos_avail, num))
bos = [ComboPlayerSet(x) for x in all]
get = []
for x in bos:
if x.total_cost / num < limit_cost:
get.append(x)
return get
def next(self, n=1):
while n:
res = defaultdict(int)
for i, j in combos(self.seq, 2):
res[i + j] += 1
ans = min([k for k in res.keys() if res[k] == 1 and k > max(self.seq)])
self.seq.append(ans)
n -= 1
return ans
def __init__(self, categories):
super().__init__(categories)
self.pairs = list(combos(self.categories, 2)) # this comes up a lot
self.grids = {n: {} for n in self.categories}
for f1, f2 in self.pairs:
g = np.empty((self.num_items, self.num_items), dtype='object')
# Both entries point to different views of the same matrix
self.grids[f1][f2] = g
self.grids[f2][f1] = g.T
def listijs(n): return np.array(#indexing
list(combos(xrange(n),2)) #first ninej are the combos
+list(izip( xrange(n),xrange(n) ))
,dtype=mui
)
@autojit
def exclude(self, *vals):
'''Indicate that these values are mutually exclusive.
This marks all intersections of values in vals as False.
'''
infos = map(self.get_info, vals)
for i1, i2 in combos(infos, 2):
if i1.cat == i2.cat:
continue
self.grids[i1.cat][i2.cat][i1.idx, i2.idx] = False
def solvable(fractals, s):
for combo in combos(range(len(fractals[0])), s):
isSolution = True
for f in fractals:
if all(f[c] == "L" for c in combo):
isSolution = False
break
if isSolution:
return combo
def combine_features(Features, Min, Max):
'''
For a list of features return every possible combination of those features from "Min" to Max".
So, if the list is [a,b,c,d], and you provide Min=1 and Max=3, this function would return:
a, b, c, d, ab, ac, ad, bc, bd, cd, abc, abd, acd, bcd.
'''
Combinations = []
[Combinations.extend(combos(Features, Feature)) for Feature in range(Min, Max + 1)]
return sorted(Combinations)
def build_dict(num):
from itertools import combinations_with_replacement as combos
from itertools import permutations
L = ['A','C','G','T']
dkeys = list(combos(L,num))
for key in dkeys:
perms_k = permutations(key)
for perm in perms_k:
if perm not in dkeys:
dkeys.append(perm)
d = {''.join(dkeys[i]): 0 for i in range(len(dkeys))}
return d
def mk_exclude(self, *vals, **cats):
keys = [self.get_info(k) for k in vals]
x = []
for a, b in combos(keys, 2):
if a.cat == b.cat:
continue
x.append(~self.vars[a.fullname, b.fullname])
for xcat, vals in cats.items():
for val in vals:
for k in keys:
x.append(self.vars[k, xcat] != val)
return z3.And(x)
def require(self, *vals):
'''Indicate that these values must go together
This marks all intersections of values in vals as True.
It will complain if two values are from the same category, as requiring
both would mean there are no solutions to the grid.
'''
infos = map(self.get_info, vals)
for i1, i2 in combos(infos, 2):
if i1.cat == i2.cat:
raise ValueError("Cannot require both {} and {}".format(
i1, i2))
self.grids[i1.cat][i2.cat][i1.idx, i2.idx] = True
def combine_features(Features, Min, Max):
'''
For a list of features return every possible combination of those features from "Min" to Max".
So, if the list is [a,b,c,d], and you provide Min=1 and Max=3, this function would return:
a, b, c, d, ab, ac, ad, bc, bd, cd, abc, abd, acd, bcd.
'''
Combinations = []
[
Combinations.extend(combos(Features, Feature))
for Feature in range(Min, Max + 1)
]
return sorted(Combinations)
def do_final_validation():
n = len(vote_db)
indices = sort_indices_by_votes(vote_db)
tuples = list(combos(np.arange(n), 2))
final_votes = np.zeros(n)
for ci, cj in tuples[::-1]:
final_votes = validate_pair(0, final_votes, indices, ci, cj)
sort = np.argsort(final_votes)[::-1]
top_scorers = {}
top20_first_round = []
for k in range(n):
top = []
for l in range(20):
c_id = indices[k][l]
coords = pdb.get_lunar_coordinates(c_id)
radius = pdb.get_real_radius(c_id)
votes = vote_db[k][c_id]
top.append({'coords': coords, 'radius': radius, 'votes': votes})
top20_first_round.append(top)
clustering_candidates = np.zeros((n, 2))
for e, i in enumerate(sort):
crater_id = indices[i][0]
coords = pdb.get_lunar_coordinates(indices[i][0])
radius = pdb.get_real_radius(indices[i][0])
votes = final_votes[i]
clustering_candidates[e] = coords
top_scorers[crater_id] = {
'coords': coords,
'radius': radius,
'votes': votes
}
print("crater ", i, ": ", coords, ' with ', votes, ' votes!')
if verbose: pdb.check_vote_db(vote_db)
if not relative_search and n != 0:
# try to find clusters
eps = 1.2 * kmpp * w_max / 1737.5
dbscan_results, cls = cluster.perform_clustering(
clustering_candidates, eps)
clusters = []
for i in mode(dbscan_results.labels_[np.where(
dbscan_results.labels_ != -1)]):
clusters.append(np.where(dbscan_results.labels_ == i))
for i in clusters:
print(clustering_candidates[i])
return top_scorers, top20_first_round
def calcEdges(top, bot, top_label, bot_label, isomorphic=False):
start = time.time()
edges = []
top_idxs = list(range(len(top)))
check_range = range(1, len(top) + 1)
for checkCount in check_range:
for checkIdxs in combos(top_idxs, checkCount):
checks_idxs_pairs = findReducingSet(checkIdxs, top, bot,
foxhole_graph)
edges += createEdges(checks_idxs_pairs, top_label, bot_label,
checkIdxs)
if isomorphic:
edges += switchPartLabels(edges)
end = time.time()
print(f'calcEdges time: {end-start:.5}')
return edges
def construct_mp_graph(filename, eventNames, minSamplingFreq, maxPWeight):
tables = []
geneSets, freqs = [], []
N = 0
count = 0
maxRWeight = 0
passPoint = 0
with open(filename) as f:
for l in f:
if l.startswith("#") or float(l.split("\t")[1]) < maxPWeight:
continue
if float(l.split("\t")[1]) > maxRWeight:
maxRWeight = float(l.split("\t")[1])
passPoint = sum([
nCr(len(G.split(", ")), 2)
for G in l.rstrip().split("\t")[3::3]
])
count += 1
arr = l.rstrip().split("\t")
freq = float(arr[0])
geneSets.append([G.split(", ") for G in arr[3::3]])
freqs.append(freq)
if freq >= minSamplingFreq:
tables.append(create_table_row(arr, eventNames))
# After loading the sets that will go into the graph,
# compute edges between each pair of genes.
edges = defaultdict(float)
N = sum(freqs)
for Gs, freq in zip(geneSets, freqs):
for G in Gs:
for g1, g2 in combos(G, 2):
if g1 in eventNames and g2 in eventNames:
edges[frozenset([g1, g2])] += freq / N
# Create a graph from the edge list
mpgEdges = [(list(S)[0], list(S)[1], dict(weight=w))
for S, w in edges.iteritems()]
MPG = nx.Graph()
MPG.add_edges_from(mpgEdges)
return MPG, tables, passPoint
def construct_mp_graph(resultsTable, eventNames, minSamplingFreq, maxPWeight):
""" Construct marginal prob. graph from collections with weight larger than permuted data (maxPWeight) """
""" Using genesets with the maximum weight to generate expected number of nodes in the graph for delta selection """
tables = []
geneSets, freqs = [], []
N = 0
maxRWeight = 0
expectedPoint = 0
for l in resultsTable:
if l.startswith("#") or float(l.split("\t")[1]) < maxPWeight: continue
if float(l.split("\t")[1]) > maxRWeight:
maxRWeight = float(l.split("\t")[1])
expectedPoint = sum([
nCr(len(G.split(", ")), 2)
for G in l.rstrip().split("\t")[3::3]
])
arr = l.rstrip().split("\t")
freq = float(arr[0])
geneSets.append([G.split(", ") for G in arr[3::3]])
freqs.append(freq)
if freq >= minSamplingFreq:
tables.append(create_table_row(arr, eventNames))
# After loading the sets that will go into the graph,
# compute edges between each pair of genes.
edges = defaultdict(float)
N = sum(freqs)
for Gs, freq in zip(geneSets, freqs):
for G in Gs:
for g1, g2 in combos(G, 2):
if g1 in eventNames and g2 in eventNames:
edges[frozenset([g1, g2])] += freq / N
# Create a graph from the edge list
mpgEdges = [(list(S)[0], list(S)[1], dict(weight=w))
for S, w in edges.iteritems()]
MPG = nx.Graph()
MPG.add_edges_from(mpgEdges)
return MPG, tables, expectedPoint
def construct_mp_graph(filename, eventNames, minSamplingFreq, maxPWeight):
""" Construct marginal prob. graph from collections with weight larger than permuted data (maxPWeight) """
""" Using genesets with the maximum weight to generate expected number of nodes in the graph for delta selection """
tables = []
geneSets, freqs = [], []
N = 0
maxRWeight = 0
expectedPoint = 0
with open(filename) as f:
for l in f:
if l.startswith("#") or float(l.split("\t")[1]) < maxPWeight: continue
if float(l.split("\t")[1]) > maxRWeight:
maxRWeight = float(l.split("\t")[1])
expectedPoint = sum([nCr(len(G.split(", ")),2) for G in l.rstrip().split("\t")[3::3] ])
arr = l.rstrip().split("\t")
freq = float(arr[0])
geneSets.append([G.split(", ") for G in arr[3::3] ])
freqs.append(freq)
if freq >= minSamplingFreq:
tables.append(create_table_row(arr, eventNames))
# After loading the sets that will go into the graph,
# compute edges between each pair of genes.
edges = defaultdict(float)
N = sum(freqs)
for Gs, freq in zip(geneSets, freqs):
for G in Gs:
for g1, g2 in combos(G, 2):
if g1 in eventNames and g2 in eventNames:
edges[frozenset([g1, g2])] += freq/N
# Create a graph from the edge list
mpgEdges = [ (list(S)[0], list(S)[1], dict(weight=w)) for S, w in edges.iteritems() ]
MPG = nx.Graph()
MPG.add_edges_from(mpgEdges)
return MPG, tables, expectedPoint
def construct_mp_graph(filename, eventNames, minSamplingFreq, maxPWeight):
tables = []
geneSets, freqs = [], []
N = 0
count = 0
maxRWeight = 0
passPoint = 0
with open(filename) as f:
for l in f:
if l.startswith("#") or float(l.split("\t")[1]) < maxPWeight: continue
if float(l.split("\t")[1]) > maxRWeight:
maxRWeight = float(l.split("\t")[1])
passPoint = sum([nCr(len(G.split(", ")),2) for G in l.rstrip().split("\t")[3::3] ])
count+=1
arr = l.rstrip().split("\t")
freq = float(arr[0])
geneSets.append([G.split(", ") for G in arr[3::3] ])
freqs.append(freq)
if freq >= minSamplingFreq:
tables.append(create_table_row(arr, eventNames))
# After loading the sets that will go into the graph,
# compute edges between each pair of genes.
edges = defaultdict(float)
N = sum(freqs)
for Gs, freq in zip(geneSets, freqs):
for G in Gs:
for g1, g2 in combos(G, 2):
if g1 in eventNames and g2 in eventNames:
edges[frozenset([g1, g2])] += freq/N
# Create a graph from the edge list
mpgEdges = [ (list(S)[0], list(S)[1], dict(weight=w)) for S, w in edges.iteritems() ]
MPG = nx.Graph()
MPG.add_edges_from(mpgEdges)
return MPG, tables, passPoint
def geometric_voting_abs(boxes, km_pp, r_tol, d_tol, wmax, verb=False):
global radius_candidates, rtol, dtol, centers, vote_db, kmpp, w_max, relative_search, verbose
relative_search = False
vote_db = {}
radius_candidates = {}
dtol = d_tol
rtol = r_tol
w_max = wmax
kmpp = km_pp
n = boxes.shape[0]
radii = np.zeros(n)
center_points = np.zeros((n, 2))
# calculate all radii and center points
for i, box in enumerate(boxes):
radii[i] = util.calculate_radius(box) * kmpp
center_points[i] = util.calculate_center(box)
# get sorting vector
sort = radii.argsort()[::-1]
# sort dat shit
radii = radii[sort]
center_points = center_points[sort]
center_points *= kmpp
centers = center_points
# get the radius candidates
for c, r in enumerate(radii):
radius_candidates[c] = pdb.get_craters_by_real_radius(r, r_tol)
tuples = list(combos(np.arange(n), 2))
t = len(tuples)
intersect = False
for count, (ci, cj) in enumerate(tuples):
util.print_progress(count, t, 'Pairs loop', 'complete')
ci_candidates, cj_candidates = search_pair(ci, cj, False)
vote_db = add_votes_to_db(vote_db, ci, ci_candidates)
vote_db = add_votes_to_db(vote_db, cj, cj_candidates)
if not intersect:
vote_db = add_votes_to_db(vote_db, ci, radius_candidates[ci])
vote_db = add_votes_to_db(vote_db, cj, radius_candidates[cj])
# check_db()
return do_final_validation()
def rank_actors(filename="top250movies.txt", epsilon=0.85):
"""Read the specified file and construct a graph where node a points to
node b with weight w if actor a and actor b were in w movies together but
actor b was listed first. Use NetworkX to compute the PageRank values of
the actors, then rank them with get_ranks().
Each line of the file has the format
title/actor1/actor2/actor3/...
meaning actor2 and actor3 should each have an edge pointing to actor1,
and actor3 should have an edge pointing to actor2.
"""
movies = nx.DiGraph()
data = []
actors = set()
with open(filename,"r",encoding = "utf-8") as file:
for line in file.readlines():
#format the data into a list of lists, where each list is a movie
line = line.strip().split('/')
#delete the movie titles. Irrelevant info.
delete = line.pop(0)
data.append(line)
#get a comprehensive list of actors
for line in data:
for actor in line:
#comprehensive lists is a set to avoid repeats
actors.add(actor)
actors = list(actors) #there are almost 15k actors
# n = len(actors)
#construct graph with edges
for line in data:
#combos is an iterator object so must be accessed in this way
for i in combos(line,2):
high_paid, low_paid = i
#edges point to higher-billed actors
if movies.has_edge(low_paid,high_paid) is True:
movies[low_paid][high_paid]["weight"] += 1
else:
movies.add_edge(low_paid,high_paid,weight = 1)
return get_ranks(nx.pagerank(movies,alpha = epsilon))
def get_validated_genesets(large_set,small_set):
"""
Using size n gs to validate n+1 gs.
1. get all combination in large gs
2. if all combinations in size_n, keep it, otherwise through this size_n+1
3. cb contains all combinations, if keep size_n+1, cb will be saved otherwise not.
"""
start = time.time()
ssize = len (next(iter(small_set.values()))['genes'])
keep = []
keep_cb = set()
for ls in large_set:
cb = combos(large_set[ls]['genes'], ssize)
cb = list(cb)
save = True
for i in cb:
i = ' '.join(i)
if i not in small_set:
save = False
break
if save:
# save all the possible combination to a set
keep_cb |= set([' '.join(x) for x in cb])
keep.append(ls)
end = time.time()
print("[valid size = %d] get_validated_genesets time = %d"%(ssize+1,(end-start)/60))
if len(keep)==0:
print ("[valid size = %d] nothing left!"%(ssize+1))
return None
else:
out = MyDict()
print ("[valid size = %d] %d pass validation !"%(ssize+1,len(keep)))
for i in keep:
out[i] = {"tg":large_set[i]['tg'],'genes':large_set[i]['genes']}
#sorted(out.keys(), key=lambda S: out[S]["tg"], reverse=True)
return {"comb": keep_cb,"valid":out}
def measure(anns, kmpp):
hdf = pd.HDFStore('crater_database/crater_db.h5', 'r')
crater_db = hdf.get('/db')
g = len(anns)
tuples = combos(np.arange(g), 2)
u = 0
v = 0
diff_dist = 0
diff_rad = 0
for a in range(g):
box_a = [anns[a]['x1'], anns[a]['y1'], anns[a]['x2'], anns[a]['y2']]
pixel_derived_radius = calculate_radius(box_a) * kmpp
real_radius = crater_db.loc[crater_db.index[(anns[a]['crater_id'])], 'radius']
d = real_radius / pixel_derived_radius
diff_rad += d
print('RADIUS DIFF: ', d)
v += 1
diff_rad /= v
print('########################### RADIUS AVG DIFF: ', diff_rad)
for a, b in tuples:
u += 1
print('#####################')
print('distances between ', anns[a]['crater_id'], ' with ', anns[a]['lat'], anns[a]['lon'], 'and',
anns[b]['crater_id'], ' with ', anns[b]['lat'], anns[b]['lon'])
hav = calculate_haversine_distance([anns[a]['lat'], anns[a]['lon']], [anns[b]['lat'], anns[b]['lon']])
appr_real = approximate_visual_distance(hav)
print('haversine: ', hav)
print('approx: ', appr_real)
box_a = [anns[a]['x1'], anns[a]['y1'], anns[a]['x2'], anns[a]['y2']]
box_b = [anns[b]['x1'], anns[b]['y1'], anns[b]['x2'], anns[b]['y2']]
approx_dist_pixels = \
cdist(np.array([calculate_center(box_a)]), np.array([calculate_center(box_b)]))[0][0]
appr_img = approx_dist_pixels * kmpp
print('approx pixel dist from bounding box: ', approx_dist_pixels, 'approx km dist from bounding box: ',
appr_img)
print('difference factor: ', appr_img / appr_real)
diff_dist += appr_img / appr_real
print('#####################', 'DIFF_AVG:', diff_dist / u, '####################')
hdf.close()
def exclusion_by_intersection_abs(boxes,
km_pp,
r_tol,
d_tol,
wmax,
verb=False):
global radius_candidates, rtol, dtol, centers, intersect_db, kmpp, w_max, relative_search, verbose
relative_search = False
intersect_db = {}
radius_candidates = {}
dtol = d_tol
rtol = r_tol
w_max = wmax
kmpp = km_pp
n = boxes.shape[0]
radii = np.zeros(n)
center_points = np.zeros((n, 2))
# calculate all radii and center points
for i, box in enumerate(boxes):
radii[i] = util.calculate_radius(box) * kmpp
center_points[i] = util.calculate_center(box)
# get sorting vector
sort = radii.argsort()[::-1]
# sort dat shit
radii = radii[sort]
center_points = center_points[sort]
center_points *= kmpp
centers = center_points
# get the radius candidates
for c, r in enumerate(radii):
radius_candidates[c] = pdb.get_craters_by_real_radius(r, r_tol)
tuples = list(combos(np.arange(n), 2))
for ci, cj in tuples:
ci_candidates, cj_candidates = search_pair(ci, cj, True)
add_ids_to_db(intersect_db, ci, ci_candidates)
add_ids_to_db(intersect_db, cj, cj_candidates)
return do_final_validation()
from itertools import combinations as combos
import numpy as np
def pentagonal(n):
return (n * ((3 * n) - 1) / 2)
# def pentagonal_list(num):
# return [pentagonal(j) for j in range(1, num+1)]
if __name__ == '__main__':
upper_index_bound = 2000
indices_list = []
for pair in combos(range(1, upper_index_bound), 2):
indices_list.append((pair[0], pair[1], abs(pair[0] - pair[1])))
numpairs = len(indices_list)
pentaglist = [pentagonal(j) for j in range(1, upper_index_bound)]
refined_pentagonals = [
] # List that will contain pairs of pentagonal numbers that satisfy said criteria
pentarr = np.zeros((3, numpairs), dtype=int)
for j in range(numpairs):
tup = indices_list[j]
pentarr[0, j] = tup[0]
pentarr[1, j] = tup[1]
pentarr[2, j] = tup[2]
# sort by the pentagonal number index difference
pentarr = pentarr[:, pentarr[2].argsort()]
def __init__(self, n, k, d, r, FF):
self.rref = True
self.r = r
self.n = n
self.k = k
self.FF = FF
self.q = FF.q
self.m = FF.m
self.p = FF.p
# Initialize matrix
matrix = Matrix()
F_basis = []
# Will continue until the number of independent elements are equal to d
while len(F_basis) < d:
# Create a copy of the matrix
matrix_temp = matrix.copy()
# Create a random element in F_q^m
rndElem = rndElem = random.randint(0, self.p - 1)
# Insert the random element in extension field format as a column
matrix_temp = matrix_temp.col_insert(
len(matrix_temp), Matrix(self.FF.ext([rndElem], True)))
# If the rank has increased add the element to the F_basis list
if matrix_temp.rank() > len(F_basis):
F_basis.append(rndElem)
# Keep the matrix until the next iteration
matrix = matrix_temp.copy()
# Define the F basis as a class variable to be used later
self.F_basis = F_basis
# Transpose basis matrix for future matrix multiplication
matrix = matrix.T
# Generate a parity check matrix with random elements from all_elements
# This matrix is completely randomized
self.H = []
self.H_d = []
self.G = []
# p is a static integer representing the dimension of parity-check columns
p = self.n - (self.n - self.k)
for i in range(p):
col_d = []
col = []
for j in range(self.n - self.k):
# Generate random element
rndCoeffs, rndElem = self.rndElem(matrix, d)
# Add coefficients of random element to column
col_d.append(rndCoeffs)
# Add random element to column
col.append(rndElem)
self.H_d.append(col_d)
self.H.append(col)
self.G.append(copy.deepcopy(col))
# Create an identity matrix using rnd_diag element (rnd_diag representing 1)
for i in range(self.n - self.k):
rnd_diag = []
while rnd_diag == []:
# Create random element to be used as diagonal in H matrix
rnd_diag_d, rnd_diag = self.rndElem(matrix, d)
empty_col = [[] for j in range(self.n - self.k)]
empty_col[i] = rnd_diag.copy()
empty_col_d = [[0 for i in range(d)]
for j in range(self.n - self.k)]
empty_col_d[i] = rnd_diag_d.copy()
self.H_d.append(empty_col_d)
self.H.append(empty_col)
# Transpose to the correct format
self.H_d = self.FF.transpose(self.H_d)
# Start making the generator matrix
# The n-k last columns of G are the negated transposed first k columns of H
for i in range(len(self.G)):
for j in range(len(self.G[0])):
# Multiply by the inverse of diagonal element in H
if len(self.G[i][j]):
coeff = [(self.G[i][j][0] - self.H[p + j][j][0]) % self.p]
else:
coeff = []
self.G[i][j] = self.FF.add([], coeff, "-")
# Fill in the first k columns with the identity matrix of size k x k
for i in range(self.k):
for j in range(self.k):
if i == self.k - j - 1:
self.G[i].insert(0, [0])
else:
self.G[i].insert(0, [])
# Initialize empty d*r*(n-k) x nr matrix
self.H_Fq = [[0] * self.n * r
for i in range(d * r * (self.n - self.k))]
# Iterate through rows of H keeping row iterator variable row_i
for row_i in range(len(self.H_d)):
# Iterate through the row keeping column iterator variable col_i
for col_i in range(len(self.H_d[0])):
F_element = self.H_d[row_i][col_i].copy()
for i in range(d):
# Skip d rows for each new H element, skip r rows for each coefficient
# of F basis representation of F_element
row_index = i * r + row_i * r * d
for j in range(r):
# Skip r columns for each new element while itertively moving one
# step along columns
col_index = col_i * r + j
self.H_Fq[row_index + j][col_index] = F_element[i]
# Find rd x rd invertible submatrix
if not self.rref:
tot_len = r * d * (self.n - self.k)
combinations = combos([i for i in range(tot_len)], n * r)
for combination in combinations:
H_Fq_i = []
for row_i in combination:
H_Fq_i.append(self.H_Fq[row_i])
matrix = Matrix(H_Fq_i)
if matrix.rank() == n * r:
self.H_inv = self.FF.inv_q(H_Fq_i)
self.H_inv = Matrix(self.H_inv)
self.s_is = combination
break
Project Euler Problem #34
==========================
145 is a curious number, as 1! + 4! + 5! = 1 + 24 + 120 = 145.
Find the sum of all numbers which are equal to the sum of the factorial of
their digits.
Note: as 1! = 1 and 2! = 2 are not sums they are not included.
"""
from utils import *
from math import factorial
from itertools import combinations_with_replacement as combos
factorials = [factorial(i) for i in xrange(10)]
good = []
for n in xrange(2, 7):
for combo in combos(range(10), n):
s = sum((factorials[i] for i in combo))
digits = to_digits(s)
if len(digits) != n: continue
if combo == tuple(sorted(digits)):
good.append(s)
print sum(good)
def f(g,l,n):
count = 0
for combo in combos(range(1,l+1),n):
if g<=gcd(combo) and l>=lcm(combo):
count +=1
return count
while n:
s += fact[n%10]
n /= 10
return s
def appearences(n):
s = list(set(n))
x = fact[len(n)]
for c in s:
x /= fact[n.count(c)]
return x
t = clock()
count = 0
chains = defaultdict(int)
for n in combos('0123456789', 6):
o = n = int(''.join(n))
chain, chain_len = [], 0
while True:
if chains[n] == 0:
chains[n] = None
chain_len += 1
chain.append(n)
else:
if chains[n]:
chain_len += chains[n]
break
n = fact_sum(n)
if chain_len == target:
o = str(o)
from time import clock
from itertools import combinations_with_replacement as combos
e = 5
t = clock()
total = -1
table = dict([(str(x), x ** e) for x in range(10)])
for combo in combos('0123456789', e + 1):
squareSum = sum(table[n] for n in combo)
if sorted(str(squareSum).zfill(e + 1)) == list(combo):
total += squareSum
print(total, clock() - t)
return 6 if sum(roll)%6 == 0 else sum(roll)%6
def toTwoDice(roll):
pass
roll = list(diceRoll())
# print("Roll: {0} | Sum: {1}".format(roll,sum(roll)))
# print("One Dice Map: {0}".format(toOneDice(roll)))
numDice = 2
# print("Sum Range: {0}".format(sumRange(numDice)))
# -------------------------------------------------------
# Probabilities stuff
# -------------------------------------------------------
perm = list(combos(range(1,7),3))
def findTotalCombos():
# for SUM in range(3,19):
# print(sum([len(set(permutations(s))) for s in filter((lambda x: sum(x)==SUM),perm)]))
a = [sum([len(set(permutations(s))) for s in filter((lambda x: sum(x)==SUM),perm)]) for SUM in range(3,19)]
return a
stat = findTotalCombos()
# statM = map((lambda x: 6 if x%6 == 0 else x%6),stat)
# dist = list(reduce((lambda x,y: (x[0],x[1]+y[1])),zip(statM,stat)))
sumMod11 = [x%11+2 for x in range(3,19)]
def listToDict(list):
# Takes list of tuples of size 2
# Uses first element of each tuple as key
# Adds the other elements to the value
# Accounts for "duplicate keys" unlike dict(list)
A perfect number is a number for which the sum of its proper divisors is exactly equal
to the number. For example, the sum of the proper divisors of 28 would be 1 + 2 + 4 + 7 + 14 = 28,
which means that 28 is a perfect number.
A number n is called deficient if the sum of its proper divisors is less than n and it is
called abundant if this sum exceeds n.
As 12 is the smallest abundant number, 1 + 2 + 3 + 4 + 6 = 16, the smallest number that can
be written as the sum of two abundant numbers is 24. By mathematical analysis, it can be shown
that all integers greater than 28123 can be written as the sum of two abundant numbers. However,
this upper limit cannot be reduced any further by analysis even though it is known that the greatest
number that cannot be expressed as the sum of two abundant numbers is less than this limit.
Find the sum of all the positive integers which cannot be written as the sum of two abundant numbers.
"""
from itertools import (combinations_with_replacement as combos,
ifilter,
starmap)
from operator import add
from util import sum_of_divisors
min, max = 1, 28123 # Exclude 0, specify upper bounds per problem
all = set(xrange(min, max)) # Set of all numbers [min, max)
# Find all abundant numbers <= max
abundant_nums = list(ifilter(lambda n: sum_of_divisors(n) > n, xrange(min, max)))
# Find the set of all unique sums of abundant numbers where sum(n) <= max. Exclude from all nums
print sum(all - set(ifilter(lambda n: n <= max, set(starmap(add, combos(abundant_nums, 2))))))
from itertools import permutations as combos
with open('./Gonduls/17/input.txt', 'r') as inputf:
lista = inputf.read().split('\n')
close_points = list(set(combos([0, 0, 0, 1, 1, 1, -1, -1, -1], 3)))
close_points.remove((0, 0, 0))
def return_close(position):
answer = []
for point in close_points:
answer.append((position[0] + point[0], position[1] + point[1],
position[2] + point[2]))
return answer
def new_diz(old):
#have to add outer layer of points close to old points to not repeat operation of creating new list twice
momentary = []
for el in old:
momentary += return_close(el)
for point in set(momentary):
if (point not in old.keys()):
old[point] = '.'
#creating new list of points
new = {}
for el in old:
close = return_close(el)
active = 0
for point in close:
if (point in old.keys() and old[point] == '#'):