def matchNames(self, externalNames, GsNames): """Returns a dictionnary: External name -> Gradesource name""" cprint('Matching students', 'yellow') mapping = {} # externalDict: "alphonse blah jean" -> "Jean-Alphonse Blah" externalDict = {} for n in externalNames: subNames = n subNames = subNames.replace('-', ' ').replace(',', ' ').lower().split(' ') subNames.sort() externalDict[" ".join(subNames).strip()] = n # GsDict: # "alphonse blah jean", # "blah jean", # "alphonse jean", # "alphonse blah", # -> "Jean-Alphonse Blah" GsDict = {} for n in GsNames: subNames = n subNames = subNames.replace('-', ' ').replace(',', ' ').lower().split(' ') # only keep more-than-one-letter subnames subNames = [x for x in subNames if len(x) > 1] subNames.sort() # include all possibility for possibility in utils.powerset(subNames): if len(possibility) > 1: GsDict[" ".join(possibility)] = n format = "%3s %25s - %-25s (%s)" cprint(format % ("#", "External name", "Gradesource name", "distance"), 'yellow') for i, externalCleaned in enumerate(externalDict): if externalCleaned in GsDict: dist = 0 GsCleaned = externalCleaned else: dist, GsCleaned = min([(utils.editDist(externalCleaned, x), x) for x in GsDict.keys()]) if dist == 0: mapping[externalDict[externalCleaned]] = GsDict[GsCleaned] for possibility in utils.powerset(GsCleaned.split(' ')): if len(possibility) > 1: del GsDict[' '.join(possibility)] if dist == 0: color = 'white' else: color = 'red' cprint(format % (i + 1, externalCleaned, GsCleaned, str(dist)), color) return mapping
def define_lexicon(player=[], shot=[], worlds=[]): D_et = powerset(player+shot) relational_hit = [[w, x, y] for w, x, y in product(worlds, player, shot) if y in shot[: w[player.index(x)]]] lex = { # Concessions to tractability -- these are defined extensionally (invariant across worlds): "some": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) > 0], "exactly_one": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) == 1], "every": [[X, Y] for X, Y in product(D_et, repeat=2) if set(X) <= set(Y)], "no": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) == 0], "PlayerA": [X for X in powerset(player) if a in X], "PlayerB": [X for X in powerset(player) if b in X], "PlayerC": [X for X in powerset(player) if c in X], # Tempting to intensionalize these, but that means using intensional quantifiers, # which are intractable on this set-theoretic formulation. Our goal is to understand # refinement and lexical uncertainty, which we can study using verbs and extensional # quantifiers, so this limitation seems well worth it. "player": player, "shot": shot, # Intensional predicates: "scored": [[w, x] for w, x in product(worlds, player) if len(shot[: w[player.index(x)]]) > 0], "aced": [[w, x] for w, x in product(worlds, player) if len(shot[: w[player.index(x)]]) > 1], "missed": [[w, x] for w, x in product(worlds, player) if len(shot[: w[player.index(x)]]) == 0], "hit" : [[w, x, y] for w, x, y in product(worlds, player, shot) if y in shot[: w[player.index(x)]]], # More concessions to tractability -- we'll refine these rather than the determiners; # this should have no effect because of the limited class of predicates -- no predicate # is true of both players and shots, and player and shot have the same extensions in all # worlds. "some_player": [Y for Y in powerset(player) if len(set(player) & set(Y)) > 0], "some_shot": [Y for Y in powerset(shot) if len(set(shot) & set(Y)) > 0], "exactly_one_player": [Y for Y in powerset(player) if len(set(player) & set(Y)) == 1], "exactly_one_shot": [Y for Y in D_et if len(set(shot) & set(Y)) == 1], "every_player": [Y for Y in D_et if set(player) <= set(Y)], "every_shot": [Y for Y in D_et if set(shot) <= set(Y)], "no_player": [Y for Y in D_et if len(set(player) & set(Y)) == 0], "no_shot": [Y for Y in D_et if len(set(shot) & set(Y)) == 0], # Mainly for specifying refinements: "not_every_player": [Y for Y in D_et if not(set(player) <= set(Y))], "not_every_shot": [Y for Y in D_et if not(set(shot) <= set(Y))], "scored_not_aced": [[w, x] for w, x in product(worlds, player) if len(shot[: w[player.index(x)]]) == 1], "only_PlayerA": [X for X in powerset(player) if a in X and len(X) == 1], "only_PlayerB": [X for X in powerset(player) if b in X and len(X) == 1], "only_PlayerC": [X for X in powerset(player) if c in X and len(X) == 1] } return lex
def all_hypotheses(examples): """Builds a list of all the possible hypotheses""" values = values_table(examples) h_powerset = powerset(values.keys()) hypotheses = [] for s in h_powerset: hypotheses.extend(build_attr_combinations(s, values)) hypotheses.extend(build_h_combinations(hypotheses)) return hypotheses
def prog(limit, primes): p = frozenset(primes) for i in primes: digits = tuple(digit_iter(i)) maxdig = max(digits) positions = powerset(range(len(digits))) next(positions) for j in positions: if same_digit(digits, j): dig = digits[j[0]] if prime_replace_part(digits, j, dig, p) == 8: return i
def build_h_combinations(hypotheses): """Given a set of hypotheses, builds and returns all the combinations of the hypotheses.""" h = [] h_powerset = powerset(range(len(hypotheses))) for s in h_powerset: t = [] for i in s: t.extend(hypotheses[i]) h.append(t) return h
def partition(iterable, num=1): seq = tuple(iterable) if num <= 1: yield [seq] raise StopIteration if not seq: yield [()] * num raise StopIteration for head in powerset(seq[:len(seq) - 1]): tail = drop_parallel(seq, head) for lst in partition(tail, num - 1): lst.insert(0, head) yield lst
def generalizations(examples_so_far, h): """Generalize the hypothesis. First delete operations (including disjunctions) from the hypothesis. Then, add OR operations.""" hypotheses = [] # Delete disjunctions disj_powerset = powerset(range(len(h))) for disjs in disj_powerset: h2 = h.copy() for d in reversed(list(disjs)): del h2[d] if check_all_consistency(examples_so_far, h2): hypotheses += h2 # Delete AND operations in disjunctions for i, disj in enumerate(h): a_powerset = powerset(disj.keys()) for attrs in a_powerset: h2 = h[i].copy() for a in attrs: del h2[a] if check_all_consistency(examples_so_far, [h2]): h3 = h.copy() h3[i] = h2.copy() hypotheses += h3 # Add OR operations if hypotheses == [] or hypotheses == [{}]: hypotheses = add_or(examples_so_far, h) else: hypotheses.extend(add_or(examples_so_far, h)) shuffle(hypotheses) return hypotheses
def add_or(examples_so_far, h): """Adds an OR operation to the hypothesis. The AND operations in the disjunction are generated by the last example (which is the problematic one).""" ors = [] e = examples_so_far[-1] attrs = {k: v for k, v in e.items() if k != 'GOAL'} a_powerset = powerset(attrs.keys()) for c in a_powerset: h2 = {} for k in c: h2[k] = attrs[k] if check_negative_consistency(examples_so_far, h2): h3 = h.copy() h3.append(h2) ors.append(h3) return ors
def main(domain, parameter_set, atol, rtol, min_success_rate, min_r2, test_override): valid_pools = [] for attack_set in utils.powerset([ 'bim', 'brendel', 'carlini', 'deepfool', 'fast_gradient', 'pgd', 'uniform' ], False): pool_result = get_r2(domain, parameter_set, atol, rtol, test_override, attack_set) (success_mean, _), (_, _), (r2_mean, _) = pool_result if success_mean >= min_success_rate and r2_mean >= min_r2: valid_pools.append((attack_set, pool_result)) min_size = min([len(attack_set) for attack_set, _ in valid_pools]) valid_pools = [(attack_set, pool) for attack_set, pool in valid_pools if len(attack_set) == min_size] valid_pools.sort(key=lambda x: pool_selector(x[1]), reverse=True) print(valid_pools)
def run(): """ Brute force over all the primes, retroactively incrementing family member count of all related numbers. Using a estimated upper bound of a million. """ N = 10 ** 6 primes = get_prime_sieve(N) counter = Counter(track_max=True) powersets = [list(powerset(range(1, n + 1))) for n in range(6)] pattern = None for p, is_prime in enumerate(primes): if not is_prime or p < 10: continue s = str(p) for digit in '0123456789': # No need to replace the last digit due to parity num_appear = s[:-1].count(digit) if num_appear == 0: continue replace_positions = powersets[num_appear] for positions in replace_positions: if len(positions) == 0: continue replaced = s for index, position in enumerate(positions): replaced = replacenth(replaced, digit, '*', position - index) counter.add(replaced) if counter.get_most()[1] == 8: pattern = counter.get_most()[0] # replace all the *'s with the lowest possible digit and return replace_digit = '1' if pattern[0] == '*' else '0' return int(pattern.replace('*', replace_digit))
def compute_ca_kadditive_imbfs(n, k, indir): global count global variables global pset global kadditivity indir = indir + "/" if indir is not None else None variables = frozenset([(i + 1) for i in range(n)]) pset = set([frozenset(p) for p in powerset(variables)]) kadditivity = int(k) t1 = time.time() print("Number of CPUs: %d" % multiprocessing.cpu_count()) pool = multiprocessing.Pool() n = 0 for f in os.listdir(indir): if f.endswith(".pkl.bz2") is False: continue pool.apply_async(compute_ca_kadditive_mbfs, (indir + f, ), callback=compute_nonkadditive_mbfs_cb) pool.close() pool.join() print("0/1 loss min: %.02f (%03d %%)" % (loss_min, ((100 * loss_min) / len(pset)))) print("0/1 loss max: %.02f (%03d %%)" % (loss_max, ((100 * loss_max) / len(pset)))) print("0/1 loss avg: %.02f (%03d %%)" % ((loss_avg / count_nmbfs), (100 * (loss_avg) / len(pset) / count_nmbfs))) print("total time: %.02f seconds" % (time.time() - t1))
def main(domain, parameter_set, atol, rtol, min_success_rate, test_override): valid_pools = [] for attack_set in utils.powerset([ 'bim', 'brendel', 'carlini', 'deepfool', 'fast_gradient', 'pgd', 'uniform' ], False): pool_result = get_r2(domain, parameter_set, atol, rtol, test_override, attack_set) (success_mean, _), (_, _), (r2_mean, r2_std) = pool_result if success_mean >= min_success_rate: valid_pools.append((attack_set, pool_result)) csv = 'Pool Size;R2;Error\n' for i in range(max([len(attack_set) for attack_set, _ in valid_pools])): pool_size = i + 1 print(f'Pool size: {pool_size}') chosen_pools = [(attack_set, pool) for attack_set, pool in valid_pools if len(attack_set) == pool_size] chosen_pools.sort(key=lambda x: pool_selector(x[1]), reverse=True) best_poool_attacks, best_pool_results = chosen_pools[0] print(best_poool_attacks) print(best_pool_results) (_, _), (_, _), (r2_mean, r2_std) = best_pool_results csv += f'{pool_size};{r2_mean};{r2_std}\n' csv_path = f'analysis/pools/{domain}-{parameter_set}.csv' Path(csv_path).parent.mkdir(parents=True, exist_ok=True) Path(csv_path).write_text(csv)
def compare_models(burn): """Comparison of all models.""" print burn depends_on = ['a', 't', 'v', 'z'] all_includes = powerset(['st', 'sv', 'sz']) results = {} for includes in all_includes: __dir = pj(_dir, str(burn)) if len(includes) == 0: _includes = 'none' else: _includes = ','.join(includes) if not exists(__dir): print 'hello' makedirs(__dir) path = pj(_dir, str(burn), _includes) if not exists(path): m = load_model_with_all_samples(depends_on, includes, burn) print_model_stats(m, path) x = float([l for l in open(path).readlines() if 'DIC' in l][0].split(' ')[1].replace('\n', '')) results[_includes] = x df = pd.DataFrame.from_dict(results, orient='index') print df df.columns = [burn] print df return df
def test_powerset(self): self.assertEqual(list(powerset([1, 2, 3], 1)), [(1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)])
def define_lexicon(player=[], shot=[], worlds=[]): D_et = powerset(player + shot) relational_hit = [[w, x, y] for w, x, y in product(worlds, player, shot) if y in shot[:w[player.index(x)]]] lex = { # Concessions to tractability -- these are defined extensionally (invariant across worlds): "some": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) > 0], "exactly_one": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) == 1], "every": [[X, Y] for X, Y in product(D_et, repeat=2) if set(X) <= set(Y)], "no": [[X, Y] for X, Y in product(D_et, repeat=2) if len(set(X) & set(Y)) == 0], "PlayerA": [X for X in powerset(player) if a in X], "PlayerB": [X for X in powerset(player) if b in X], "PlayerC": [X for X in powerset(player) if c in X], # Tempting to intensionalize these, but that means using intensional quantifiers, # which are intractable on this set-theoretic formulation. Our goal is to understand # refinement and lexical uncertainty, which we can study using verbs and extensional # quantifiers, so this limitation seems well worth it. "player": player, "shot": shot, # Intensional predicates: "scored": [[w, x] for w, x in product(worlds, player) if len(shot[:w[player.index(x)]]) > 0], "aced": [[w, x] for w, x in product(worlds, player) if len(shot[:w[player.index(x)]]) > 1], "missed": [[w, x] for w, x in product(worlds, player) if len(shot[:w[player.index(x)]]) == 0], "hit": [[w, x, y] for w, x, y in product(worlds, player, shot) if y in shot[:w[player.index(x)]]], # More concessions to tractability -- we'll refine these rather than the determiners; # this should have no effect because of the limited class of predicates -- no predicate # is true of both players and shots, and player and shot have the same extensions in all # worlds. "some_player": [Y for Y in powerset(player) if len(set(player) & set(Y)) > 0], "some_shot": [Y for Y in powerset(shot) if len(set(shot) & set(Y)) > 0], "exactly_one_player": [Y for Y in powerset(player) if len(set(player) & set(Y)) == 1], "exactly_one_shot": [Y for Y in D_et if len(set(shot) & set(Y)) == 1], "every_player": [Y for Y in D_et if set(player) <= set(Y)], "every_shot": [Y for Y in D_et if set(shot) <= set(Y)], "no_player": [Y for Y in D_et if len(set(player) & set(Y)) == 0], "no_shot": [Y for Y in D_et if len(set(shot) & set(Y)) == 0], # Mainly for specifying refinements: "not_every_player": [Y for Y in D_et if not (set(player) <= set(Y))], "not_every_shot": [Y for Y in D_et if not (set(shot) <= set(Y))], "scored_not_aced": [[w, x] for w, x in product(worlds, player) if len(shot[:w[player.index(x)]]) == 1], "only_PlayerA": [X for X in powerset(player) if a in X and len(X) == 1], "only_PlayerB": [X for X in powerset(player) if b in X and len(X) == 1], "only_PlayerC": [X for X in powerset(player) if c in X and len(X) == 1] } return lex
def refinements(self, semval): return powerset(semval, minsize=1)
for fmin in fmins: if coalition >= fmin: is_good = True if is_good is True: good.add(coalition) else: bad.add(coalition) return frozenset(good), frozenset(bad) def mbf_ca_kadditive(mbf, kadditivity, variables, pset): good, bad = generate_binary_table(pset, mbf) n = cplex_lbda_minmax(variables, good, bad, kadditivity) return n if __name__ == "__main__": from utils import mbf_to_str kadditivity = 1 n = 4 variables = frozenset([(i + 1) for i in range(n)]) pset = set([frozenset(p) for p in powerset(variables)]) mbf = frozenset([frozenset([1, 2]), frozenset([1, 4]), frozenset([3, 4])]) n = mbf_ca_kadditive(mbf, kadditivity, variables, pset) print("%s: %d/%d are not representable with %d-additive weights" % (mbf_to_str(mbf), n, len(pset), kadditivity))
async def play_word(self): """ Waits the delay, finds a word and plays it. """ self.thinking = True await asyncio.sleep(settings.BOT_DELAY) self.thinking = False words = self.game.all_words free = self.game.free_tiles top_word = "" max_points = 0 pid = 0 index = 0 used_chars = () if self.interrupted: self.interrupted = False self.thinking = False return # try to augment existing words for (word, player_id, i) in words: for combo in list(utils.powerset(free))[1:]: combined = word + ''.join(combo) key = ''.join(sorted(combined)) if key in self.dictionary: potential_word = random.choice(self.dictionary[key]) points = self.calculate_points( potential_word, 0 if player_id != self.id else len(word)) if points > max_points: print(potential_word) top_word, max_points = potential_word, points pid, index, used_chars = player_id, i, combo # check free chars for combo in list(utils.powerset(free))[1:]: combined = ''.join(combo) key = ''.join(sorted(combined)) if key in self.dictionary: potential_word = random.choice(self.dictionary[key]) points = self.calculate_points(potential_word, 0) if points > max_points: print(potential_word) top_word, max_points = potential_word, points pid, index, used_chars = 0, -1, combo ########## if not top_word: return if pid != -1: # delete used_chars from free for c in used_chars: self.game.free_tiles.remove(c) # update scores new_word_length = len(top_word) self.score += new_word_length if pid != 0 and pid != self.id: old_word_length = len(self.game.players[pid].words[index]) self.game.players[pid].score -= old_word_length elif pid != 0 and pid == self.id: old_word_length = len(self.words[index]) self.score -= old_word_length # update the player word lists self.words.append(top_word) if pid != 0: if pid == self.id: del self.words[index] else: del self.game.players[pid].words[index] self.game.update_play_field() self.game.send_all("valid_word", top_word, self.name)
def proposal_state_payoff(fixed_types, game, state, proposer, ret_proposal=False): ''' :param fixed_types: :param game: :param state: :param proposer: :return: a dictionary with key = agent, value = payoff - if some proposals are equally good, the proposer RANDOMLY picks one! - If agent believe that a coalition C is more available than the continuation, she will propose the greedy division for that coalition - Otherwise, she would propose a proportional weighting division ''' # next state without any coalition formed next_state_no_C = State(state.active_agents, state.t + 1, state.horizon) continuation_payoff = game_state_payoff(fixed_types, game, next_state_no_C) other_players = [ player for player in state.active_agents if player != proposer ] best_proposals = [(None, float('-inf'), None, None) ] # [C, proposer_payoff, payoff, V_C] # look at all the coalitions involving proposer for s in powerset(other_players): proposal = [proposer] + list(s) logging.debug('entertaining proposal={}'.format(proposal)) W = [fixed_types[p] for p in proposal] V_C = eval_coalition(W, game.tasks) tot_continuation = sum([continuation_payoff[p] for p in s]) # if feasible, then greedy if V_C >= tot_continuation: payoff = greedy_division(proposer, proposal, V_C, \ tot_continuation, continuation_payoff) # else, proportional weight division else: # there should always exist at least one feasible proposal: # i.e. the singleton coalition! # if not feasible, agent should just propose the singleton coalition!! logging.debug('proposal={} NOT feasible'.format(proposal)) #payoff = {agent:fixed_types[agent]/sum(W) for agent in proposal} #payoff[proposer] = 0 continue proposer_payoff = payoff[proposer] logging.debug('other_players={}, V_C={}, cont={}, payoff={}'.format(s, V_C, tot_continuation, \ proposer_payoff)) # update best proposals if proposer_payoff == best_proposals[0][1]: best_proposals.append((proposal, proposer_payoff, payoff, V_C)) elif proposer_payoff > best_proposals[0][1]: best_proposals = [(proposal, proposer_payoff, payoff, V_C)] logging.debug('best_proposals={}'.format(best_proposals)) # randomly choose the optimal proposal best_proposal, _, best_payoff_C, best_V_C = best_proposals[ np.random.choice(len(best_proposals))] active_agents = state.active_agents.difference(best_proposal) # next_state: increment time and update active_agents! next_state = State(active_agents, state.t + 1, state.horizon) payoff = game_state_payoff(fixed_types, game, next_state) res = Counter(payoff) + Counter(best_payoff_C) logging.debug('proposal_state_payoff({}, t={}, active={}, proposer={})={} | proposal chosen:{}'.format(fixed_types, state.t, \ state.active_agents, proposer, res, best_proposal)) if ret_proposal: return best_proposal, best_V_C, best_payoff_C return res
def run(self, data, y_all, d_atts, unid, expdir, feateng_type, \ env_atts_types): rawres_fname = os.path.join(expdir, 'rawres_{}.json'.format(unid)) #Set allowable datts as PCPs allowed_datts = {cat:d_atts[cat] for cat in d_atts.keys() \ if cat not in env_atts_types} #Generate Environments (assuming only cat vars) e_ins_store = eproc.get_environments(data, \ {cat:d_atts[cat] for cat in env_atts_types}) logging.info('{} environment attributes'.format(len(e_ins_store))) logging.debug('Environment attributes are {}'.format( \ str([str(e) for e in e_ins_store]))) #Now start enumerating PCPs full_res = {} with open(rawres_fname, mode='w+') as rawres: for subset in tqdm(powerset(allowed_datts.keys()), \ desc='pcp_sets', total=len(list( \ powerset(allowed_datts.keys())))): #powerset #Setup raw result logging full_res[str(subset)] = {} #Check for empty set if not subset: continue #Linear regression on all data regressors = self.get_data_regressors(allowed_datts, subset, \ feateng_type, data) x_s = data[list(itertools.chain(regressors))] reg = LinearRegression(fit_intercept=False).fit(x_s.values, \ y_all.values) #Use the normalized e_ins to compute the residuals + #Find p_values for every environment assert len(e_ins_store.keys()) > 1 #For validation performance for env in e_ins_store: e_in = e_ins_store[env] e_out = np.logical_not(e_in) #No data from environment if (e_in.sum() < 2) or (len(e_out) - e_out.sum() < 2): raise Exception('Not enough data in environment') res_in = (y_all.loc[e_in].values - reg.predict(\ x_s.loc[e_in].values)).ravel() res_out = (y_all.loc[e_out].values - reg.predict(x_s.loc[e_out].values)).ravel() #Check for NaNs if (self.mean_var_test(res_in, res_out) is np.nan) or \ (self.mean_var_test(res_in, res_out) != \ self.mean_var_test(res_in, res_out)): full_res[str(subset)][str(env)] = 'NaN' else: full_res[str(subset)][str(env)] = \ self.mean_var_test(res_in, res_out) # # TODO: Jonas uses "min(p_values) * len(environments) - 1" full_res[str(subset)]['Final_tstat'] = min( \ [p for p in full_res[str(subset)].values() \ if not isinstance(p, str)]) * \ len(e_ins_store.keys()) logging.info('Enumerated all steps') #Save results json.dump(full_res, rawres, indent=4, separators=(',', ':'))