def dict_for_p_parents(parents, categories, CPT, names, n, CPT_dict, variable):
combinations = []
parent_names = [names[i] for i in parents]
for cats in [categories[i] for i in parents]:
this_cat = []
for c in range(65, 65 + cats):
this_cat.append(chr(c))
combinations.append(this_cat)
oi_la = list(itertools.product(*combinations))
for total_count, comb in enumerate(oi_la):
name_list = [[v, c] for v, c in zip(parent_names, comb)]
node_cat_dict = {}
for i, domain in enumerate(CPT[n][total_count]):
node_cat_dict.update({chr(65 + i): domain})
if not variable in CPT_dict:
CPT_dict[variable] = [[name_list, node_cat_dict]]
else:
CPT_dict[variable].append([name_list, node_cat_dict])
return CPT_dict
def generate_sequences_for_r(r, okutsu_vars, hs):
#r = [3, 2]
seq = [3, 2, 2, 1, 1, 6, 3, 1, 1, 2, 1, 9, 1, 1, 12]
pools = []
for r_s in r:
pools += [list(okutsu_vars) for i in range(0, r_s-1)]
pools.append([list(p) for p in product([1], [1], hs)])
#print pools
#pools = [list(okutsu_vars) for i in range(0, sum(r))]
total = 0
valid = 0
combinations = []
seqs = 0
all_indco = all_indco_for_r(r)
okutsu_combs = prod([len(p) for p in pools])
print "Okutsu combinations per level: %d" % (len(okutsu_vars),)
print "Okutsu total combinations: %d" % (okutsu_combs,)
print "Index of coincidense combinations: %d" % (len(all_indco),)
for seq in product(*pools):
seq = list(seq)
types = types_from_sequence(r, list(seq))
if types is False:
continue
for indco in all_indco:
all_hidden = all_hidden_for_types_indco(types, indco)
if total == 0:
print "Max hidden slope combinations: %d" % (len(all_hidden),)
print "Max total combinations: %d" % (len(all_hidden)*len(all_indco)*okutsu_combs,)
for hidden in all_hidden:
inv = {
'j': indco,
'hidden': hidden,
'types': types,
}
total += 1
try:
verify_inv(inv)
except ValueError, e:
continue
if re.match(r'^[1-9][0]+$', str(valid)):
print "Prepared: %d" % (valid,)
combinations.append(inv)
valid += 1
def build_matrices(mat, possibilities, combinations):
assert len(mat) == (DIM_Y-len(possibilities))
if len(possibilities) == 0:
combinations.append(mat)
return
for possibility in possibilities[0]:
if len(possibilities) == DIM_Y:
mat = [possibility]
else:
mat.append(possibility)
build_matrices(list(mat), possibilities[1:], combinations)
mat.pop()
return
def cor_features_output():
feat_df = dict.fromkeys([feat.attribute for feat in FEATURES],
pd.DataFrame())
for feat in FEATURES:
ports = pd.read_csv(path_join(PATH_EVAL, 'eval', feat.attribute,
'separated', PERIOD, T, N_MIN, N_DAYS,
'score', 'csv'),
sep=';',
index_col=0)
feat_df[feat.attribute] = ports.applymap(sign_to_score)
combinations = [FEATURES]
labels = ['all']
for feat in FEATURES:
labels.append(feat.attribute)
temp = FEATURES[:]
temp.remove(feat)
combinations.append(temp)
threshold_ano, all_ano = (dict.fromkeys([str(l) for l in combinations], [])
for i in range(2))
for l in combinations:
result = pd.DataFrame()
for feat in l:
result = result.add(feat_df[feat.attribute], fill_value=0)
result = result.astype(int)
ind_thr, ind_all = ([] for i in range(2))
for port, row in result.iterrows():
for i, date in enumerate(DATES[N_DAYS:]):
ind_all.append('|'.join([str(port), date, str(row[i])]))
if row[i] > T_ANO:
ind_thr.append('|'.join([str(port), date, str(row[i])]))
threshold_ano[str(l)] = ind_thr
all_ano[str(l)] = ind_all
print(threshold_ano.values())
unique_ano = set([
'|'.join(el.split('|')[:-1]) for thr in threshold_ano.values()
for el in thr
])
final_array = pd.DataFrame(index=unique_ano,
columns=[str(l) for l in combinations],
dtype=np.int8)
for value in unique_ano:
for l in combinations:
for anomaly in all_ano[str(l)]:
if value in anomaly:
final_array.loc[value, str(l)] = int(anomaly.split('|')[2])
break
else:
final_array.loc[value, str(l)] = 0
fig, axis = plt.subplots()
final = np.array(final_array, dtype=int)
image = axis.imshow(final, cmap='YlOrRd')
axis.set_xticks(np.arange(len(combinations)))
axis.set_yticks(np.arange(len(unique_ano)))
axis.set_xticklabels(labels)
axis.set_yticklabels([
an.split('|')[0] + ' - ' + an.split('|')[1][0:2] + '/' +
an.split('|')[1][2:] for an in unique_ano
])
axis.tick_params(axis='both', which='major', labelsize=5)
plt.setp(axis.get_xticklabels(),
rotation=35,
ha='right',
rotation_mode='anchor')
for i in range(len(unique_ano)):
for j in range(len(combinations)):
color = 'b' if final[i, j] > T_ANO else 'c'
text = axis.text(j,
i,
final[i, j],
ha='center',
va='center',
color=color,
size=5)
axis.set_title('Intensity of anomalies with features varying', size=5)
fig.savefig(path_join(PATH_FIGURES, 'cor_features', T, N_MIN, N_DAYS,
PERIOD, 'png'),
dpi=600,
bbox_inches='tight')
for i, l in enumerate(combinations[1:]):
rho_s, p_s = [
round(val * 100, 1)
for val in spearmanr(final_array.iloc[:, 0], final_array[str(l)])
]
print(
labels[i + 1],
int(sum(np.subtract(final_array.iloc[:, 0], final_array[str(l)]))),
rho_s)
def rule_to_combination(union_rule): combinations = [] for item in union_rule: combinations.append(transfer_np_to_arr(item)) return combinations
combinations = []
while set_size < 5:
set_size += 1
for index_combination in index_combinations:
for index in range(len(primes_cache)):
index_combo = index_combination.__add__([index])
combo = itemgetter(*index_combo)(primes_cache)
permutations_of_primes = permutations(combo, 2)
all_prime = True
for pair in permutations_of_primes:
if not is_prime(concatenate_numbers(pair)):
all_prime = False
break
if all_prime and set(index_combo) not in combinations:
combinations.append(set(index_combo))
print(set_size)
print(combinations)
index_combinations = [list(combination) for combination in combinations]
combinations = []
for combination in index_combinations:
combo = itemgetter(*combination)(primes_cache)
print(combo)
print(sum(combo))
filtration_values1 = np.array(filtration_values1)
K1 = K1[order1]
filtration_values1 = filtration_values1[order1]
face_index_dictionary1 = {face: index for index, face in enumerate(K1)}
#BOUNDARY MATRIX
m = len(K1)
partial1 = np.zeros((m, m))
for j in range(0, m):
if len(K1[j]) == 1:
continue
else:
combinations = []
for facet in itertools.combinations(K1[j], len(K1[j]) - 1):
if frozenset(facet) in K1:
combinations.append(frozenset(facet))
boundary1 = list(face_index_dictionary1[frozenset(facet)]
for facet in combinations)
for i in range(0, m):
if face_index_dictionary1[K1[i]] in boundary1:
partial1[i][j] = 1
else:
continue
#PERSISTENT 0-HOMOLOGY:
pers0_homology = dict()
for i in range(len(S)):
if i in persistent_homology(partial)[1]:
x = persistent_homology(partial)[1].index(i)
pers0_homology[i] = K[x]
else:
def main():
'''
Basic worflow:
Load Top X Selective Primers
Populate Locations of Primers
Score Combinations For All Sizes
'''
global fg_genome_length
global bg_genome_length
global seq_ends
global output_file
global score_func
parser = argparse.ArgumentParser(description="score mers")
parser.add_argument("-f", "--foreground", help="foreground fasta file", required=True)
parser.add_argument("-b", "--background", help="background fasta file", required=True)
parser.add_argument("-o", "--output", help="output fasta with UIDs in the file", required=True)
parser.add_argument("-s", "--selectivity-file", help="mer selectivity file generated by select_mers.py", required=False)
parser.add_argument("-c", "--combination-file", help="a set of combinations you want to score", required=False)
parser.add_argument("-m", "--mer-file", help="a set of you want to score all combinations of", required=False)
parser.add_argument("-r", "--rescore-file", help="rescore an already scored output file", required=False)
args = parser.parse_args()
nb_flags = len(filter(lambda x: x is None, [args.combination_file, args.selectivity_file,args.mer_file, args.rescore_file]))
if nb_flags != 3:
if nb_flags == 4:
parser.error("you must have at least one input file to score from [-s -c -m -r]")
else:
parser.error("you can only have one input file to score from" )
exit(1)
if not os.path.isfile(args.foreground):
parser.error(args.foreground + " not found")
if not os.path.isfile(args.background):
parser.error(args.background + " not found")
output_file = args.output
print "Getting genome length"
fg_genome_length = get_length(args.foreground)
bg_genome_length = get_length(args.background)
print "fg_genome_length:", fg_genome_length
print "bg_genome_length:", bg_genome_length
print "Populating sequence end points"
seq_ends = load_end_points(args.foreground)
if args.selectivity_file is not None:
print "Scoring all mer combinations"
selectivity_fh = open(args.selectivity_file, "r")
# load our mer list into python
mer_selectivity = selectivity_fh.readlines()
mer_selectivity = [ x for x in mer_selectivity if not x.startswith('#')]
# get the last max_check (it's sorted)
if len(mer_selectivity) > max_check:
selected_mers = mer_selectivity[-max_check:]
else:
selected_mers = mer_selectivity
# load it into our fg and bg counts into their dictionaries
for mer in selected_mers:
split_mer = mer.split()
fg_mers[split_mer[0]] = []
bg_mers[split_mer[0]] = int(split_mer[2])
selected_mers = [x.split()[0] for x in selected_mers]
if len(selected_mers) is 0:
print "no mers found."
exit(1)
# we already have our background counts
initialize_mers(args.foreground, args.background, load_background=False)
print "Scoring mer combinations"
score_all_combinations(selected_mers)
elif args.combination_file is not None:
print "Scoring specific mer combinations"
combinations = []
combination_fh = open(args.combination_file, "r")
for line in combination_fh:
if line.startswith("#"):
continue
mers = line.split()
combinations.append(mers)
for mer in mers:
fg_mers[mer] = []
bg_mers[mer] = []
if len(combinations) is 0:
print "no combinations found."
exit(1)
initialize_mers(args.foreground, args.background)
score_specific_combinations(combinations)
elif args.mer_file is not None:
print "Scoring all possible mer combinations from ", args.mer_file
mer_fh = open(args.mer_file, "r")
for mer in mer_fh:
if mer.startswith("#"):
continue
mer = mer.strip()
if(len(mer.split()) > 1):
print "skipping line:", mer, "each line should contain only one mer"
continue
fg_mers[mer] = []
bg_mers[mer] = []
if len(fg_mers.keys()) is 0:
print "no mers found."
exit(1)
initialize_mers(args.foreground, args.background)
score_all_combinations(fg_mers.keys())
elif args.rescore_file is not None:
print "Scoring all mer combinations from ", args.rescore_file
combinations = []
score_fh = open(args.rescore_file, "r")
for line in score_fh:
if line.startswith("#"):
continue
split_line = line.split('\t')
combination = split_line[1].split()
combinations.append(combination)
for mer in combination:
fg_mers[mer] = []
bg_mers[mer] = []
if len(combinations) is 0:
print "no combinations found."
exit(1)
initialize_mers(args.foreground, args.background)
print "re-scoring scores file"
score_specific_combinations(combinations)
print "output file:", output_file
def main():
"""
Creates the traces and computes the differences
"""
# Creating the parser
parser = create_argparse()
args = parser.parse_args()
analysis_method = args.analysis_method
if analysis_method not in analysis_methods:
print('Error: The fingerprint method is incorrect,' +
'choose one of the following: ' + str(analysis_methods))
exit(0)
# Creating and configuring the logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)s - %(levelname)s: %(message)s')
file_handler = logging.FileHandler('run.log')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.info(
'=============================================================')
logger.info('Starting')
pcap_dir = args.pcap_directory
# We compute the number of pcap files in the directory
pcap_file_num = compute_files_dir(pcap_dir)
print('Number of pcap files: ' + str(pcap_file_num))
# List that will contains the traces objects
traces = make_traces(pcap_dir, pcap_file_num)
logger.debug('# of trace file = %s', len(traces))
# We perform the fingerprint and comparisons according to the argument
if analysis_method == 'all':
all_sign_traces = []
# We perform all the relevant sign methods to then compare the results
for sign_meth in relevant_sign_methods:
fingerprint(traces, sign_meth)
comparisons = compare_traces(traces, True)
all_sign_traces.append(comparisons)
# We flatten the list completely (twice the op is necessary to do so)
all_sign_traces = [i for sublist in all_sign_traces for i in sublist]
all_sign_traces = [i for sublist in all_sign_traces for i in sublist]
assembled_sim = assemble_similarities(all_sign_traces)
#display_info(traces)
display_assembled_similarities(assembled_sim)
print('\n')
display_links_strength(all_sign_traces)
print('\n')
display_csuites_inter(traces)
print('\n')
#display_random_similarities(traces)
elif analysis_method == 'combinations':
# We perform fingerprinting of all combinations of the attributes
# Computing nCr to get the result of 2 amongst pcap_file_num
f = math.factorial
nCr = f(pcap_file_num) / f(2) / f(pcap_file_num - 2)
print('nCr =', nCr)
# Try out several percentages
percentage_down = 0
percentage_up = 0
down = (nCr * percentage_down / 100)
up = nCr - (nCr * percentage_up / 100)
print('down =', down)
print('up =', up)
# Creating the combinations and fingerprinting them
attr_list = TLSClientHello.attr_for_fpt
all_sign_traces = []
combinations = []
total_combinations = 0
# Iterating through all combinations of client hello attributes
for L in range(0, len(attr_list) + 1):
for attr_sublist in itertools.combinations(attr_list, L):
fingerprint_for_combinations(traces, attr_sublist)
comparisons = compare_traces(traces, True)
# Adding only relevant information by checking on the size
comp_size = len(comparisons)
if (comp_size >= down and comp_size <= up):
all_sign_traces.append(comparisons)
combinations.append(attr_sublist)
# Counting the total number of combinations
total_combinations += 1
# Displaying the "winning" combinations of attributes
#print(combinations)
print('number of winning combinations =', len(combinations), 'out of',
total_combinations)
# We flatten the list completely (twice the op is necessary to do so)
all_sign_traces = [i for sublist in all_sign_traces for i in sublist]
all_sign_traces = [i for sublist in all_sign_traces for i in sublist]
print('all_sign_traces size :', len(all_sign_traces))
print('\n')
assembled_sim = assemble_similarities(all_sign_traces)
#display_info(traces)
display_assembled_similarities(assembled_sim)
print('\n')
display_links_strength(all_sign_traces)
"""
print('\n')
display_csuites_inter(traces)
"""
elif analysis_method == 'libs_compare':
# Here we only compare the iot devices with the tls libraries
# We produce a heatmap in the end to show which devices in closest to
# which tls library
libs_directory = '../captures/ssl_libs' # Hard coded value to change
libs_file_num = compute_files_dir(libs_directory)
libs_traces = make_traces(libs_directory, libs_file_num)
# Computing nCr to get the result of 2 amongst pcap_file_num
f = math.factorial
nCr = f(pcap_file_num) / f(2) / f(pcap_file_num - 2)
print('nCr =', nCr)
# Try out several percentages
percentage_down = 0
percentage_up = 0
down = (nCr * percentage_down / 100)
up = nCr - (nCr * percentage_up / 100)
print('down =', down)
print('up =', up)
# Creating the combinations and fingerprinting them
attr_list = TLSClientHello.attr_for_fpt
combinations = []
total_combinations = 0
final_matrix = np.zeros(shape=(len(traces), len(libs_traces)))
# Iterating through all combinations of client hello attributes
for L in range(0, len(attr_list) + 1):
for attr_sublist in itertools.combinations(attr_list, L):
fingerprint_for_combinations(traces, attr_sublist)
fingerprint_for_combinations(libs_traces, attr_sublist)
(matrix,
comp_num) = compare_devices_to_libs(traces, libs_traces)
# Adding only relevant information to the final_matrix
# by checking on the size
if (comp_num > down and comp_num < up):
final_matrix += matrix
combinations.append(attr_sublist)
# Counting the total number of combinations
total_combinations += 1
print('total_combinations =', total_combinations)
print('number of fingerprint method used =', len(combinations))
print(final_matrix)
"""
final_matrix = np.zeros(shape=(len(traces), len(libs_traces)))
for sign_meth in relevant_sign_methods:
fingerprint(traces, sign_meth)
fingerprint(libs_traces, sign_meth)
(matrix, comp_num) = compare_devices_to_libs(traces,
libs_traces)
final_matrix += matrix
print(final_matrix)
"""
to_delete = []
for i in range(0, final_matrix.shape[0]):
cmpt = 0
for j in range(0, final_matrix.shape[1]):
final_matrix[i][j] = (final_matrix[i][j] / len(combinations) *
100)
if final_matrix[i][j] == 0:
cmpt += 1
if cmpt == len(libs_traces):
# Adding the right index to delete (minus the number of
# already added index because on the removal the length of
# the list will decrease)
to_delete.append(i - len(to_delete))
heatmap_devices = []
heatmap_libs = []
for trace in traces:
heatmap_devices.append(trace.name)
for lib in libs_traces:
heatmap_libs.append(lib.name)
print('to_delete =', to_delete)
# Deleting selected rows in the matrix
for i in to_delete:
final_matrix = np.delete(final_matrix, (i), axis=0)
del heatmap_devices[i]
print(final_matrix)
# Creating the heatmap
create_heatmap(final_matrix, heatmap_devices, heatmap_libs)
print('heatmap_devices size =', len(heatmap_devices))
elif analysis_method == 'csuites_check':
# Looking for outdated cipher suites in the client hellos
vulnerable_csuites = []
for trace in traces:
for pkt in trace.client_hellos:
vulnerable_csuites = pkt.tls_info.check_cipher_suites()
choosed_csuites = []
for pkt in trace.server_hellos:
choosed_csuites.append(pkt.tls_info.csuite)
display_vulnerabilities(trace.name, vulnerable_csuites,
choosed_csuites)
print('Number of TLSClientHello: ', len(trace.client_hellos))
elif analysis_method == 'cert_check':
# Displaying the issuer sequence of the certificates
display_issuer_sequence(traces)
# Displaying the result of the nmap command on the servers
display_nmap_result(traces)
elif analysis_method == 'version_check':
# Displaying versions of the TLS protocol used
display_version(traces)
else:
# We perform the fingerprinting method specified in the arguments
fingerprint(traces, analysis_method)
# We compute and display the similarities
comparisons = compare_traces(traces)
# We flatten the list and sort it to then display it
comparisons = [item for sublist in comparisons for item in sublist]
comparisons.sort(key=lambda triple: triple[2])
display_info(traces)
display_similarities(comparisons)
logger.info('Finished\n\n\n\n\n')
