def dict_for_p_parents(parents, categories, CPT, names, n, CPT_dict, variable): combinations = [] parent_names = [names[i] for i in parents] for cats in [categories[i] for i in parents]: this_cat = [] for c in range(65, 65 + cats): this_cat.append(chr(c)) combinations.append(this_cat) oi_la = list(itertools.product(*combinations)) for total_count, comb in enumerate(oi_la): name_list = [[v, c] for v, c in zip(parent_names, comb)] node_cat_dict = {} for i, domain in enumerate(CPT[n][total_count]): node_cat_dict.update({chr(65 + i): domain}) if not variable in CPT_dict: CPT_dict[variable] = [[name_list, node_cat_dict]] else: CPT_dict[variable].append([name_list, node_cat_dict]) return CPT_dict
def generate_sequences_for_r(r, okutsu_vars, hs): #r = [3, 2] seq = [3, 2, 2, 1, 1, 6, 3, 1, 1, 2, 1, 9, 1, 1, 12] pools = [] for r_s in r: pools += [list(okutsu_vars) for i in range(0, r_s-1)] pools.append([list(p) for p in product([1], [1], hs)]) #print pools #pools = [list(okutsu_vars) for i in range(0, sum(r))] total = 0 valid = 0 combinations = [] seqs = 0 all_indco = all_indco_for_r(r) okutsu_combs = prod([len(p) for p in pools]) print "Okutsu combinations per level: %d" % (len(okutsu_vars),) print "Okutsu total combinations: %d" % (okutsu_combs,) print "Index of coincidense combinations: %d" % (len(all_indco),) for seq in product(*pools): seq = list(seq) types = types_from_sequence(r, list(seq)) if types is False: continue for indco in all_indco: all_hidden = all_hidden_for_types_indco(types, indco) if total == 0: print "Max hidden slope combinations: %d" % (len(all_hidden),) print "Max total combinations: %d" % (len(all_hidden)*len(all_indco)*okutsu_combs,) for hidden in all_hidden: inv = { 'j': indco, 'hidden': hidden, 'types': types, } total += 1 try: verify_inv(inv) except ValueError, e: continue if re.match(r'^[1-9][0]+$', str(valid)): print "Prepared: %d" % (valid,) combinations.append(inv) valid += 1
def build_matrices(mat, possibilities, combinations): assert len(mat) == (DIM_Y-len(possibilities)) if len(possibilities) == 0: combinations.append(mat) return for possibility in possibilities[0]: if len(possibilities) == DIM_Y: mat = [possibility] else: mat.append(possibility) build_matrices(list(mat), possibilities[1:], combinations) mat.pop() return
def cor_features_output(): feat_df = dict.fromkeys([feat.attribute for feat in FEATURES], pd.DataFrame()) for feat in FEATURES: ports = pd.read_csv(path_join(PATH_EVAL, 'eval', feat.attribute, 'separated', PERIOD, T, N_MIN, N_DAYS, 'score', 'csv'), sep=';', index_col=0) feat_df[feat.attribute] = ports.applymap(sign_to_score) combinations = [FEATURES] labels = ['all'] for feat in FEATURES: labels.append(feat.attribute) temp = FEATURES[:] temp.remove(feat) combinations.append(temp) threshold_ano, all_ano = (dict.fromkeys([str(l) for l in combinations], []) for i in range(2)) for l in combinations: result = pd.DataFrame() for feat in l: result = result.add(feat_df[feat.attribute], fill_value=0) result = result.astype(int) ind_thr, ind_all = ([] for i in range(2)) for port, row in result.iterrows(): for i, date in enumerate(DATES[N_DAYS:]): ind_all.append('|'.join([str(port), date, str(row[i])])) if row[i] > T_ANO: ind_thr.append('|'.join([str(port), date, str(row[i])])) threshold_ano[str(l)] = ind_thr all_ano[str(l)] = ind_all print(threshold_ano.values()) unique_ano = set([ '|'.join(el.split('|')[:-1]) for thr in threshold_ano.values() for el in thr ]) final_array = pd.DataFrame(index=unique_ano, columns=[str(l) for l in combinations], dtype=np.int8) for value in unique_ano: for l in combinations: for anomaly in all_ano[str(l)]: if value in anomaly: final_array.loc[value, str(l)] = int(anomaly.split('|')[2]) break else: final_array.loc[value, str(l)] = 0 fig, axis = plt.subplots() final = np.array(final_array, dtype=int) image = axis.imshow(final, cmap='YlOrRd') axis.set_xticks(np.arange(len(combinations))) axis.set_yticks(np.arange(len(unique_ano))) axis.set_xticklabels(labels) axis.set_yticklabels([ an.split('|')[0] + ' - ' + an.split('|')[1][0:2] + '/' + an.split('|')[1][2:] for an in unique_ano ]) axis.tick_params(axis='both', which='major', labelsize=5) plt.setp(axis.get_xticklabels(), rotation=35, ha='right', rotation_mode='anchor') for i in range(len(unique_ano)): for j in range(len(combinations)): color = 'b' if final[i, j] > T_ANO else 'c' text = axis.text(j, i, final[i, j], ha='center', va='center', color=color, size=5) axis.set_title('Intensity of anomalies with features varying', size=5) fig.savefig(path_join(PATH_FIGURES, 'cor_features', T, N_MIN, N_DAYS, PERIOD, 'png'), dpi=600, bbox_inches='tight') for i, l in enumerate(combinations[1:]): rho_s, p_s = [ round(val * 100, 1) for val in spearmanr(final_array.iloc[:, 0], final_array[str(l)]) ] print( labels[i + 1], int(sum(np.subtract(final_array.iloc[:, 0], final_array[str(l)]))), rho_s)
def rule_to_combination(union_rule): combinations = [] for item in union_rule: combinations.append(transfer_np_to_arr(item)) return combinations
combinations = [] while set_size < 5: set_size += 1 for index_combination in index_combinations: for index in range(len(primes_cache)): index_combo = index_combination.__add__([index]) combo = itemgetter(*index_combo)(primes_cache) permutations_of_primes = permutations(combo, 2) all_prime = True for pair in permutations_of_primes: if not is_prime(concatenate_numbers(pair)): all_prime = False break if all_prime and set(index_combo) not in combinations: combinations.append(set(index_combo)) print(set_size) print(combinations) index_combinations = [list(combination) for combination in combinations] combinations = [] for combination in index_combinations: combo = itemgetter(*combination)(primes_cache) print(combo) print(sum(combo))
filtration_values1 = np.array(filtration_values1) K1 = K1[order1] filtration_values1 = filtration_values1[order1] face_index_dictionary1 = {face: index for index, face in enumerate(K1)} #BOUNDARY MATRIX m = len(K1) partial1 = np.zeros((m, m)) for j in range(0, m): if len(K1[j]) == 1: continue else: combinations = [] for facet in itertools.combinations(K1[j], len(K1[j]) - 1): if frozenset(facet) in K1: combinations.append(frozenset(facet)) boundary1 = list(face_index_dictionary1[frozenset(facet)] for facet in combinations) for i in range(0, m): if face_index_dictionary1[K1[i]] in boundary1: partial1[i][j] = 1 else: continue #PERSISTENT 0-HOMOLOGY: pers0_homology = dict() for i in range(len(S)): if i in persistent_homology(partial)[1]: x = persistent_homology(partial)[1].index(i) pers0_homology[i] = K[x] else:
def main(): ''' Basic worflow: Load Top X Selective Primers Populate Locations of Primers Score Combinations For All Sizes ''' global fg_genome_length global bg_genome_length global seq_ends global output_file global score_func parser = argparse.ArgumentParser(description="score mers") parser.add_argument("-f", "--foreground", help="foreground fasta file", required=True) parser.add_argument("-b", "--background", help="background fasta file", required=True) parser.add_argument("-o", "--output", help="output fasta with UIDs in the file", required=True) parser.add_argument("-s", "--selectivity-file", help="mer selectivity file generated by select_mers.py", required=False) parser.add_argument("-c", "--combination-file", help="a set of combinations you want to score", required=False) parser.add_argument("-m", "--mer-file", help="a set of you want to score all combinations of", required=False) parser.add_argument("-r", "--rescore-file", help="rescore an already scored output file", required=False) args = parser.parse_args() nb_flags = len(filter(lambda x: x is None, [args.combination_file, args.selectivity_file,args.mer_file, args.rescore_file])) if nb_flags != 3: if nb_flags == 4: parser.error("you must have at least one input file to score from [-s -c -m -r]") else: parser.error("you can only have one input file to score from" ) exit(1) if not os.path.isfile(args.foreground): parser.error(args.foreground + " not found") if not os.path.isfile(args.background): parser.error(args.background + " not found") output_file = args.output print "Getting genome length" fg_genome_length = get_length(args.foreground) bg_genome_length = get_length(args.background) print "fg_genome_length:", fg_genome_length print "bg_genome_length:", bg_genome_length print "Populating sequence end points" seq_ends = load_end_points(args.foreground) if args.selectivity_file is not None: print "Scoring all mer combinations" selectivity_fh = open(args.selectivity_file, "r") # load our mer list into python mer_selectivity = selectivity_fh.readlines() mer_selectivity = [ x for x in mer_selectivity if not x.startswith('#')] # get the last max_check (it's sorted) if len(mer_selectivity) > max_check: selected_mers = mer_selectivity[-max_check:] else: selected_mers = mer_selectivity # load it into our fg and bg counts into their dictionaries for mer in selected_mers: split_mer = mer.split() fg_mers[split_mer[0]] = [] bg_mers[split_mer[0]] = int(split_mer[2]) selected_mers = [x.split()[0] for x in selected_mers] if len(selected_mers) is 0: print "no mers found." exit(1) # we already have our background counts initialize_mers(args.foreground, args.background, load_background=False) print "Scoring mer combinations" score_all_combinations(selected_mers) elif args.combination_file is not None: print "Scoring specific mer combinations" combinations = [] combination_fh = open(args.combination_file, "r") for line in combination_fh: if line.startswith("#"): continue mers = line.split() combinations.append(mers) for mer in mers: fg_mers[mer] = [] bg_mers[mer] = [] if len(combinations) is 0: print "no combinations found." exit(1) initialize_mers(args.foreground, args.background) score_specific_combinations(combinations) elif args.mer_file is not None: print "Scoring all possible mer combinations from ", args.mer_file mer_fh = open(args.mer_file, "r") for mer in mer_fh: if mer.startswith("#"): continue mer = mer.strip() if(len(mer.split()) > 1): print "skipping line:", mer, "each line should contain only one mer" continue fg_mers[mer] = [] bg_mers[mer] = [] if len(fg_mers.keys()) is 0: print "no mers found." exit(1) initialize_mers(args.foreground, args.background) score_all_combinations(fg_mers.keys()) elif args.rescore_file is not None: print "Scoring all mer combinations from ", args.rescore_file combinations = [] score_fh = open(args.rescore_file, "r") for line in score_fh: if line.startswith("#"): continue split_line = line.split('\t') combination = split_line[1].split() combinations.append(combination) for mer in combination: fg_mers[mer] = [] bg_mers[mer] = [] if len(combinations) is 0: print "no combinations found." exit(1) initialize_mers(args.foreground, args.background) print "re-scoring scores file" score_specific_combinations(combinations) print "output file:", output_file
def main(): """ Creates the traces and computes the differences """ # Creating the parser parser = create_argparse() args = parser.parse_args() analysis_method = args.analysis_method if analysis_method not in analysis_methods: print('Error: The fingerprint method is incorrect,' + 'choose one of the following: ' + str(analysis_methods)) exit(0) # Creating and configuring the logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(name)s - %(levelname)s: %(message)s') file_handler = logging.FileHandler('run.log') file_handler.setFormatter(formatter) logger.addHandler(file_handler) logger.info( '=============================================================') logger.info('Starting') pcap_dir = args.pcap_directory # We compute the number of pcap files in the directory pcap_file_num = compute_files_dir(pcap_dir) print('Number of pcap files: ' + str(pcap_file_num)) # List that will contains the traces objects traces = make_traces(pcap_dir, pcap_file_num) logger.debug('# of trace file = %s', len(traces)) # We perform the fingerprint and comparisons according to the argument if analysis_method == 'all': all_sign_traces = [] # We perform all the relevant sign methods to then compare the results for sign_meth in relevant_sign_methods: fingerprint(traces, sign_meth) comparisons = compare_traces(traces, True) all_sign_traces.append(comparisons) # We flatten the list completely (twice the op is necessary to do so) all_sign_traces = [i for sublist in all_sign_traces for i in sublist] all_sign_traces = [i for sublist in all_sign_traces for i in sublist] assembled_sim = assemble_similarities(all_sign_traces) #display_info(traces) display_assembled_similarities(assembled_sim) print('\n') display_links_strength(all_sign_traces) print('\n') display_csuites_inter(traces) print('\n') #display_random_similarities(traces) elif analysis_method == 'combinations': # We perform fingerprinting of all combinations of the attributes # Computing nCr to get the result of 2 amongst pcap_file_num f = math.factorial nCr = f(pcap_file_num) / f(2) / f(pcap_file_num - 2) print('nCr =', nCr) # Try out several percentages percentage_down = 0 percentage_up = 0 down = (nCr * percentage_down / 100) up = nCr - (nCr * percentage_up / 100) print('down =', down) print('up =', up) # Creating the combinations and fingerprinting them attr_list = TLSClientHello.attr_for_fpt all_sign_traces = [] combinations = [] total_combinations = 0 # Iterating through all combinations of client hello attributes for L in range(0, len(attr_list) + 1): for attr_sublist in itertools.combinations(attr_list, L): fingerprint_for_combinations(traces, attr_sublist) comparisons = compare_traces(traces, True) # Adding only relevant information by checking on the size comp_size = len(comparisons) if (comp_size >= down and comp_size <= up): all_sign_traces.append(comparisons) combinations.append(attr_sublist) # Counting the total number of combinations total_combinations += 1 # Displaying the "winning" combinations of attributes #print(combinations) print('number of winning combinations =', len(combinations), 'out of', total_combinations) # We flatten the list completely (twice the op is necessary to do so) all_sign_traces = [i for sublist in all_sign_traces for i in sublist] all_sign_traces = [i for sublist in all_sign_traces for i in sublist] print('all_sign_traces size :', len(all_sign_traces)) print('\n') assembled_sim = assemble_similarities(all_sign_traces) #display_info(traces) display_assembled_similarities(assembled_sim) print('\n') display_links_strength(all_sign_traces) """ print('\n') display_csuites_inter(traces) """ elif analysis_method == 'libs_compare': # Here we only compare the iot devices with the tls libraries # We produce a heatmap in the end to show which devices in closest to # which tls library libs_directory = '../captures/ssl_libs' # Hard coded value to change libs_file_num = compute_files_dir(libs_directory) libs_traces = make_traces(libs_directory, libs_file_num) # Computing nCr to get the result of 2 amongst pcap_file_num f = math.factorial nCr = f(pcap_file_num) / f(2) / f(pcap_file_num - 2) print('nCr =', nCr) # Try out several percentages percentage_down = 0 percentage_up = 0 down = (nCr * percentage_down / 100) up = nCr - (nCr * percentage_up / 100) print('down =', down) print('up =', up) # Creating the combinations and fingerprinting them attr_list = TLSClientHello.attr_for_fpt combinations = [] total_combinations = 0 final_matrix = np.zeros(shape=(len(traces), len(libs_traces))) # Iterating through all combinations of client hello attributes for L in range(0, len(attr_list) + 1): for attr_sublist in itertools.combinations(attr_list, L): fingerprint_for_combinations(traces, attr_sublist) fingerprint_for_combinations(libs_traces, attr_sublist) (matrix, comp_num) = compare_devices_to_libs(traces, libs_traces) # Adding only relevant information to the final_matrix # by checking on the size if (comp_num > down and comp_num < up): final_matrix += matrix combinations.append(attr_sublist) # Counting the total number of combinations total_combinations += 1 print('total_combinations =', total_combinations) print('number of fingerprint method used =', len(combinations)) print(final_matrix) """ final_matrix = np.zeros(shape=(len(traces), len(libs_traces))) for sign_meth in relevant_sign_methods: fingerprint(traces, sign_meth) fingerprint(libs_traces, sign_meth) (matrix, comp_num) = compare_devices_to_libs(traces, libs_traces) final_matrix += matrix print(final_matrix) """ to_delete = [] for i in range(0, final_matrix.shape[0]): cmpt = 0 for j in range(0, final_matrix.shape[1]): final_matrix[i][j] = (final_matrix[i][j] / len(combinations) * 100) if final_matrix[i][j] == 0: cmpt += 1 if cmpt == len(libs_traces): # Adding the right index to delete (minus the number of # already added index because on the removal the length of # the list will decrease) to_delete.append(i - len(to_delete)) heatmap_devices = [] heatmap_libs = [] for trace in traces: heatmap_devices.append(trace.name) for lib in libs_traces: heatmap_libs.append(lib.name) print('to_delete =', to_delete) # Deleting selected rows in the matrix for i in to_delete: final_matrix = np.delete(final_matrix, (i), axis=0) del heatmap_devices[i] print(final_matrix) # Creating the heatmap create_heatmap(final_matrix, heatmap_devices, heatmap_libs) print('heatmap_devices size =', len(heatmap_devices)) elif analysis_method == 'csuites_check': # Looking for outdated cipher suites in the client hellos vulnerable_csuites = [] for trace in traces: for pkt in trace.client_hellos: vulnerable_csuites = pkt.tls_info.check_cipher_suites() choosed_csuites = [] for pkt in trace.server_hellos: choosed_csuites.append(pkt.tls_info.csuite) display_vulnerabilities(trace.name, vulnerable_csuites, choosed_csuites) print('Number of TLSClientHello: ', len(trace.client_hellos)) elif analysis_method == 'cert_check': # Displaying the issuer sequence of the certificates display_issuer_sequence(traces) # Displaying the result of the nmap command on the servers display_nmap_result(traces) elif analysis_method == 'version_check': # Displaying versions of the TLS protocol used display_version(traces) else: # We perform the fingerprinting method specified in the arguments fingerprint(traces, analysis_method) # We compute and display the similarities comparisons = compare_traces(traces) # We flatten the list and sort it to then display it comparisons = [item for sublist in comparisons for item in sublist] comparisons.sort(key=lambda triple: triple[2]) display_info(traces) display_similarities(comparisons) logger.info('Finished\n\n\n\n\n')