def dr_cluster(X, y, X_test, dr_steps, savedir, ds): """ Apply dimensionality reduction and then KMeans and EM clustering :param X: np array, training samples :param y: np array, labels :param X_test: np.array, test data :param dr_steps: list of dimensionality reduction objects :param savedir: string, output directory :param ds: string, name of dataset :return: tuple, best clusters for each dr type """ cluster_idx = {'musk': 1, 'cancer': 0, 'shoppers': 0} best_km = [] best_em = [] best_test_km = [] best_test_em = [] for dr_step in dr_steps: km, em, km_test, em_test = A3.cluster(range(2, 21), X, y, savedir, ds, tnse_range=range(3, 5), dr_step=dr_step, X_test=X_test) best_km.append(km[cluster_idx[ds]]) best_em.append(em[cluster_idx[ds]]) best_test_km.append(km_test[cluster_idx[ds]]) best_test_em.append(em_test[cluster_idx[ds]]) return best_km, best_em, best_test_km, best_test_em
def cluster_nn(X_train, y_train, X_test, y_test, savedir, ds, cluster_type): logging.info( 'Running neural net with {} clusters as features'.format(cluster_type)) pipe = A3.baseline_ann(X_train, y_train, ds) ypred = pipe.predict(X_test) util.confusionMatrix('{}-{}-ann'.format(ds, cluster_type), y_test, ypred, savedir) return f1_score(y_test, ypred)
def sum_primes_below(n): """Compute the sum of primes below the given threshold. >>> sum_primes_below(10) 17 >>> sum_primes_below(2000000) 142913828922 """ return sum(A3.prime_generator(n))
def dr_ann(X_train, y_train, X_test, y_test, dr_steps, savedir, ds, cluster=None): if cluster is not None: c = ' with clustering from {}'.format(cluster) else: c = '' logging.info('ANN: Running baseline neural net' + c) baseline = A3.baseline_ann(X_train, y_train, ds) ypred = baseline.predict(X_test) util.confusionMatrix('{}{}-baseline'.format(ds, cluster), y_test, ypred, savedir) scores = [f1_score(y_test, ypred)] score_names = ['baseline'] for dr_step in dr_steps: drname = dr_step.__class__.__name__.lower() score_names.append(drname) logging.info('ANN: Running neural net with {} dimension reduction'. format(drname) + c) # Get trained ann with dr ann = A3.dr_ann(X_train, y_train, dr_step, ds) ypred = ann.predict(X_test) util.confusionMatrix('{}-{}{}'.format(ds, drname, cluster), y_test, ypred, savedir) scores.append(f1_score(y_test, ypred)) logging.info('ANN {} F1 Scores: {}'.format(c, scores)) util.plotBarScores(scores, score_names, ds, savedir, phaseName='{}-{}'.format(ds, cluster)) plt.close('all')
def factor_map(n): """Compute prime factors of an integer. >>> factor_map(29) {29: 1} >>> factor_map(72) {2: 3, 3: 2} """ map = {} for factor in A3.factor_list(n): map[factor] = map.get(factor, 0) + 1 return map
def run(Ex): if Ex == "1": A1.a() elif Ex == "2": A2.b() elif Ex == "3": A3.c() elif Ex == "4": A4.d() elif Ex == "5": A5.e() elif Ex == "6": A6.f() elif Ex == "7": A7.g() elif Ex == "8": A8.h() elif Ex == "9": A9.i() elif Ex == "10": A10.j() elif Ex == "intro": Aintro.A() else: print "Invalid Exercise"
def find_nth_prime(n): """Compute the nth prime number using an unbounded prime generator. >>> find_nth_prime(6) 13 >>> find_nth_prime(101) 547 >>> find_nth_prime(10001) 104743 """ primes = A3.prime_generator() for x in range(n): prime = primes.next() return prime
def computed_with_prime_powers(n): """A faster method observes that for any prime number, a simple power of that prime number will be the smallest number with the that factorization count. For example, 16 is the largest power of 2 less than or equal to 20. >>> computed_with_prime_powers(20) 232792560 >>> computed_with_prime_powers(40) 5342931457063200 >>> computed_with_prime_powers(80) 32433859254793982911622772305630400L >>> computed_with_prime_powers(160) 117671955487901874837890815641362681946988303003141220897970719568000L """ product = 1 for prime in A3.prime_generator(n): count = int(math.log(n, prime)) product *= prime ** count return product
def B3(command, subject, player_input): global success if command == 1: history.append('visited') B4.B4_text(B4.history) success = True elif command == 2 or command == 4: print delay("A wall blocks your way.") print success = False elif command == 3: history.append('visited') A3.A3_text(A3.history) elif command == 5: if subject.lower() in B3_inventory: overall.main_character.inventory[subject] = B3_inventory[subject] del B3_inventory[subject] print delay("%s has been added to inventory" % (subject)) elif subject.lower( ) in overall.final_enemy.inventory and overall.final_enemy.is_alive == False: overall.main_character.inventory[ subject] = overall.final_enemy.inventory[subject] del overall.final_enemy.inventory[subject] print delay("%s added to inventory" % (subject)) elif subject.lower( ) in overall.final_enemy.inventory and overall.final_enemy.is_alive == True and overall.final_enemy.is_asleep == True: print delay("The ENEMY is still, you can not take items from it.") elif subject.lower() in overall.main_character.inventory: print delay('%s already in inventory' % (subject)) else: print delay("Can not pick up %s" % (subject)) print sucess = False elif command == 6: print if subject.lower() in overall.main_character.inventory: B3_inventory[subject] = overall.main_character.inventory[subject] del overall.main_character.inventory[subject] delay('%s has been removed from inventory' % (subject)) else: delay('%s not in inventory' % (subject)) print success = False elif command == 7: if subject.lower() == 'enemy' and overall.if_empty( overall.final_enemy.inventory ) == False and overall.final_enemy.is_alive == False: print delay("The body of the enemy contains a: ") print for item in overall.final_enemy.inventory: print delay(item) print print elif subject.lower() == 'enemy' and overall.if_empty( overall.final_enemy.inventory ) == True and overall.final_enemy.is_alive == False: print delay("This body carries nothing on it.") print elif subject.lower( ) == 'enemy' and overall.final_enemy.is_alive == True and overall.final_enemy.is_asleep == True: print delay("This ENEMY is still alive. You can not search it.") print else: print delay("Cannot search %s" % (subject)) print print success = False elif command == 8: print if subject.lower( ) in overall.main_character.inventory and overall.main_character.inventory[ subject.lower( )].form == 'weapon' and overall.final_enemy.is_asleep == True: print delay("As you attack the ENEMY stirs from its slumber.") print overall.final_enemy.is_asleep = False B3_combat(command, subject, 0) elif subject.lower() in overall.main_character.inventory: overall.main_character.inventory[subject.lower()].use() if overall.main_character.inventory[ subject.lower()].form == 'food': del overall.main_character.inventory[subject.lower()] else: delay('No %s in inventory' % (subject)) print success = False elif command == 9: for item in overall.main_character.inventory: print delay(item) print if overall.if_empty(overall.main_character.inventory) == True: print delay("You have nothing in your inventory!") print success = False elif command == 10: print if subject.lower() in overall.main_character.inventory: overall.main_character.inventory[subject].describe() else: delay('%s not in inventory' % (subject)) print success = False elif command == 11: print overall.main_character.death() success = False else: success = False if success == False: B3_reit(nothing)
def dr(X, y, savedir, ds): # First do pca pca_pipe = A3.pca(X, y) pca = pca_pipe.named_steps['pca'] plt.savefig('{}/{}-pca.png'.format(savedir, ds)) np.savetxt('{}/{}-pca-ev.csv'.format(savedir, ds), pca.explained_variance_) np.savetxt('{}/{}-pca-ev-ratio.csv'.format(savedir, ds), pca.explained_variance_ratio_) plt.close('all') plt.plot(np.cumsum(pca.explained_variance_ratio_)) plt.xlabel('n_components') plt.ylabel('explained variance (%)') plt.savefig('{}/{}-pca-ev.png'.format(savedir, ds)) plt.close('all') reconstruction_error = A3.recon_error(pca, X) logging.info('PCA reconstruction error: {}'.format(reconstruction_error)) # second ICA ica = None max_kurtosis = -np.inf ica_range = range(10, X.shape[1], 10) kurt_per_comp = [] for i in ica_range: ica_pipe = A3.ica(X, y, i) # This should be selected by kurtosis kurt = A3.avg_kurtosis(ica_pipe.transform(X)) kurt_per_comp.append(kurt) logging.info('ICA {} average kurtosis: {}'.format(i, kurt)) if kurt > max_kurtosis: ica = ica_pipe.named_steps['fastica'] max_kurtosis = kurt logging.info('ICA max kurtosis {} with {} components'.format( max_kurtosis, ica.components_.shape[0])) plt.plot(ica_range, kurt_per_comp) plt.xlabel('n_components') plt.ylabel('mean kurtosis') plt.savefig('{}/{}-ica-kurtosis.png'.format(savedir, ds)) # RP logging.info('Starting randomized projection...') rp_errors = [] rp = None best_rp_err = np.inf reconstruction_errors = [] for rp_run in range(10): logging.info('RP iteration {}'.format(rp_run)) best_run = np.inf run_errors = [] for i in range(10, X.shape[1], 10): rp_pipe = A3.random_projection(X, y, i) err = A3.recon_error( rp_pipe.named_steps['gaussianrandomprojection'], X) run_errors.append(err) logging.info('RP {} components reconstruction error: {}'.format( i, err)) if err < best_rp_err: rp = rp_pipe.named_steps['gaussianrandomprojection'] best_rp_err = err if err < best_run: best_run = err reconstruction_errors.append(run_errors) rp_errors.append(best_run) pd.DataFrame(reconstruction_errors, columns=range(10, X.shape[1], 10)).to_csv( '{}/{}-rp-reconstruction.csv'.format(savedir, ds)) # Manually set random projection if ds == 'musk': rp.set_params(n_components=50) elif ds == 'shoppers': rp.set_params(n_components=30) plt.figure() plt.plot(range(10), rp_errors) plt.xlabel('iteration') plt.ylabel('reconstruction error') plt.savefig('{}/{}-rp-reconstruction.png'.format(savedir, ds)) plt.close('all') logging.info('RP best n_components: {}'.format(rp.n_components_)) # TODO: fourth dimension reduction rf_pipe = A3.rfselect(X, y) rf = rf_pipe.named_steps['randomforest'] return [pca, ica, rp, rf]
def main(): args = get_args() savedir = util.mktmpdir(args.outdir) # Logging copy-pasted from logging cookbook # http://docs.python.org/howto/logging-cookbook.html#logging-to-multiple-destinations logging.basicConfig(format='%(asctime)s %(message)s', filename='{}/output.log'.format(savedir), level=logging.INFO) console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger('').addHandler(console) for ds in ['musk', 'shoppers']: formatter = logging.Formatter( '{}: %(levelname)-8s %(message)s'.format(ds)) console.setFormatter(formatter) logging.info('==========Starting {} Dataset =============='.format(ds)) dataset = datajanitor.getDataset(ds) dataset.getData() x_train, x_test, y_train, y_test = \ dataset.partitionData(percent=0.3, randomState=10) # ********************* # # **** Clustering **** # # ********************* # if 'cluster' in args.phase or 'cluster-ann' in args.phase: km_train_clust, em_train_clust, km_test_clust, em_test_clust = \ A3.cluster(range(2, 21), x_train, y_train, savedir, ds, tnse_range=None, X_test=x_test) # ******************************* # # **** Clusters as features **** # # ******************************* # # one-hot encode and then add clusters to train and test features cluster_nn_scores = { 'km': [], 'em': [], 'km+em': [], 'km_only': [], 'em_only': [], 'kmem_only': [] } if 'cluster-ann' in args.phase: for i in range(5): km_x_train = add_cluster_dims(x_train, km_train_clust[i]) km_x_test = add_cluster_dims(x_test, km_test_clust[i]) em_x_train = add_cluster_dims(x_train, em_train_clust[i]) em_x_test = add_cluster_dims(x_test, em_test_clust[i]) km_score = cluster_nn(km_x_train, y_train, km_x_test, y_test, savedir, ds, 'km{}'.format(i + 2)) em_score = cluster_nn(em_x_train, y_train, em_x_test, y_test, savedir, ds, 'em{}'.format(i + 2)) kmem_x_train = add_cluster_dims(km_x_train, em_train_clust[i]) kmem_x_test = add_cluster_dims(km_x_test, em_test_clust[i]) kmem_score = cluster_nn(kmem_x_train, y_train, kmem_x_test, y_test, savedir, ds, 'kmem{}'.format(i + 2)) # do only clusters km_only = cluster_nn(km_train_clust[i].reshape(-1, 1), y_train, km_test_clust[i].reshape(-1, 1), y_test, savedir, ds, 'km_only{}'.format(i + 2)) em_only = cluster_nn(km_train_clust[i].reshape(-1, 1), y_train, km_test_clust[i].reshape(-1, 1), y_test, savedir, ds, 'em_only{}'.format(i + 2)) kmem_only = cluster_nn( np.append(km_train_clust[i].reshape(-1, 1), em_train_clust[i].reshape(-1, 1), axis=1), y_train, np.append(km_test_clust[i].reshape(-1, 1), em_test_clust[i].reshape(-1, 1), axis=1), y_test, savedir, ds, 'kmem_only{}'.format(i + 2)) util.plotBarScores([ km_score, em_score, kmem_score, km_only, em_only, kmem_only ], [ 'km-ann', 'em-ann', 'kmem-ann', 'km_only', 'em_only', 'kmem_only' ], ds, savedir, phaseName='{}-cluster-{}-ann'.format( ds, i + 2)) cluster_nn_scores['km'].append(km_score) cluster_nn_scores['em'].append(em_score) cluster_nn_scores['km+em'].append(kmem_score) cluster_nn_scores['km_only'].append(km_only) cluster_nn_scores['em_only'].append(em_only) cluster_nn_scores['kmem_only'].append(kmem_only) plt.close('all') pd.DataFrame.from_dict(data=cluster_nn_scores).to_csv( '{}/{}-clusternn.csv'.format(savedir, ds)) # ************************ # # **** Dim Reduction **** # # ************************ # # You actually have to do dimension reduction, there is no choice dr_steps = dr(x_train, y_train, savedir, ds) # *********************** # # **** DR + Cluster **** # # *********************** # if 'dr-cluster' in args.phase: km_train_clust, em_train_clust, km_test_clust, em_test_clust = \ dr_cluster(x_train, y_train, x_test, dr_steps, savedir, ds) # ******************* # # **** DR + ANN **** # # ******************* # dr_ann(x_train, y_train, x_test, y_test, dr_steps, savedir, ds)
def A2(command, subject, player_input): global success if command == 1: if 'unlocked' in history: print history.append('visited') A3.A3_text(A3.history) success = True else: print delay('This door is locked') print success = False elif command == 2: history.append('visited') B2.B2_text(B2.history) elif command == 3: print delay("A wall blocks your way") print success = False elif command == 4: history.append("visited") A1.A1_text(A1.history) elif command == 5: if subject.lower() in A2_inventory: overall.main_character.inventory[subject] = A2_inventory[subject] del A2_inventory[subject] print delay('%s has been added to inventory' %(subject)) elif subject.lower() in overall.main_character.inventory: print delay('%s already in inventory' %(subject)) else: print delay("Can not pick up %s" %(subject)) print success = False elif command == 6: print if subject.lower() in overall.main_character.inventory: A2_inventory[subject] = overall.main_character.inventory[subject] del overall.main_character.inventory[subject] delay('%s has been removed from inventory' %(subject)) else: delay('%s not in inventory' %(subject)) print success = False elif command == 7: print delay('Nothing here to search') print success = False elif command == 8: print if subject.lower() == 'iron key' and subject.lower() in overall.main_character.inventory: history.append('unlocked') delay("IRON KEY unlocks the NORTH door") print elif subject.lower() in overall.main_character.inventory: overall.main_character.inventory[subject].use() if overall.main_character.inventory[subject.lower()].form == 'food': del overall.main_character.inventory[subject.lower()] else: delay('No %s in inventory' %(subject)) print success = False elif command == 9: for item in overall.main_character.inventory: print delay(item) print if overall.if_empty(overall.main_character.inventory) == True: print delay("You have nothing in your inventory!") print success = False elif command == 10: print if subject.lower() in overall.main_character.inventory: overall.main_character.inventory[subject].describe() else: delay('%s not in inventory' %(subject)) print success = False elif command == 11: print overall.main_character.death() success = False else: success = False if success == False: A2_reit(nothing)
#!/bin/python import A3 import os path = os.getcwd() filename = 'keys.txt' path = path + '/' + filename #A3.debug = True print(A3.createDictionaries(path)) tmp = A3.encode('Hello World') print(tmp) print(A3.decode(tmp))
if result: print("A1 Suspicious") for line in result: print(line) else: print("A1 Benign") result = A2.evaluate(sourceCode) if result: print("A2 Suspicious") for line in result: print(line) else: print("A2 Benign") result = A3.evaluate(sourceCode) if result: print("A3 Suspicious") for line in result: print(line) else: print("A3 Benign") result = A4.evaluate(sourceCode) if result: print("A4 Suspicious") for line in result: print(line) else: print("A4 Benign")
import A3 import A4 flag = True epochs = 250 print('Project Υπολογιστικής Νοημοσύνης - Μέρος Α') while flag: rn = input('Επιλέξτε ερώτημα: \nA2\nA3\nA4\nA5\n\'all\'' ' to run all 7 files\n' '') if rn == 'all': single_layer.single_layer(epochs) extra_layer.extra_layer(epochs) early_stopping.early_stopping(epochs) A3.a3(epochs) A4.a4(epochs) if rn == 'A2': rn_2 = input( 'Επιλέξτε υποερώτημα: \n1 (Single-layer MLP)\n2 (Two-Layer MLP)\n3 (Early-Stopping)\n' ) if rn_2 == '1': single_layer.single_layer(epochs) flag = False if rn_2 == '2': extra_layer.extra_layer(epochs) flag = False if rn_2 == '3': early_stopping.early_stopping(epochs) flag = False if rn == 'A3':