def bwt_check_decryption():
"""
This function allows to determine the source according to the priorities
to recover the sequence to decrypt
"""
global seq_decrypt, list_decrypt, bwt, list_decrypt_step, seq_fasta, button
button = "Decryption"
if seq_fasta != "": #retrieve the file give
bwt = seq_fasta.strip()
elif path.exists("Results/encrypt_sequence.txt"): #check file exist
with open("Results/encrypt_sequence.txt", "r") as file_decrypt:
bwt = file_decrypt.readline().strip(
) #retrieve the first line of the file
elif path.exists("Results/decompress_sequence.txt"):
with open("Results/decompress_sequence.txt", "r") as file_decomp2:
bwt = file_decomp2.readline().strip(
) #retrieve the first line of the file
#This function makes it possible to recover for each of the buttons where the sequence to be analysed comes from.
if bwt != "":
if check_dollars(bwt):
seq_decrypt, list_decrypt, list_decrypt_step = decryptage(bwt)
else:
error("This sequence is not encrypt")
else:
error("The encryption file does not exist")
def duo_check_encrypt():
"""
This function allows to determine the source according to the priority
to recover the sequence to be encrypted then compressed
"""
global seq_fasta, origin_seq, button
global bwt, list_pattern, list_pattern_sort
global given_sequence, binary_dict, compress_seq, add_zero, binary_sequence
button = "Encryption/compression"
if sequence.get("1.0", "end-1c") != "":
origin_seq = sequence.get("1.0", "end-1c")
elif seq_fasta != "":
origin_seq = seq_fasta.replace("\n", "")
elif path.exists("Results/decrypt_sequence.txt"):
with open("Results/decrypt_sequence.txt", "r") as file_encrypt:
origin_seq = file_encrypt.readline().strip(
) #retrieve the first line of the file
elif path.exists("Results/decompress_decrypt_sequence.txt"):
with open("Results/decompress_decrypt_sequence.txt", "r") as file_duo:
origin_seq = file_duo.readline().strip(
) #retrieve the first line of the file
else:
error("Please give a study sequence")
#This condition makes it possible to decide whether to launch the analysis or not
if origin_seq != "":
if not check_dollars(origin_seq) and check_seq(origin_seq.upper()):
list_pattern, list_pattern_sort, bwt = cryptage(origin_seq.upper())
given_sequence, binary_dict, compress_seq, add_zero, binary_sequence = compression_huffman(
bwt)
else:
error("The sequence is not already encrypt")
def setup_clothing(size_factor, root_output, root_input):
build_gen = tuple(init_build(path.join(root_input, "data/body_parts/humanoid.fa")))
build_act = tuple(parse_actions(path.join(root_input, "data/actions/humanoid.txt")))
output_images = path.join(root_output, "clothing")
if not path.exists(output_images):
makedirs(output_images)
output_data = path.join(root_output, "clothing/data")
if not path.exists(output_data):
makedirs(output_data)
resource_image_path = path.join(root_input, "images/characters/humanoid.unnamed/")
appendage_files = {app: AppendageFile(path.join(output_data, app.bodypart + ".txt")) for app in build_gen}
complete_file = path.join(output_images, "COMPLETE")
if not path.exists(complete_file):
p = ProgressBar(len(build_act) * len(build_gen), "Making pictures...")
for app in build_gen:
# Order is preserved when called with no change in between #
joints = app.joints.keys()
coords = app.joints.values()
if app.bodypart[-2:] == "_l":
image_filename = "".join((resource_image_path, app.bodypart[:-2] + "_r", ".png"))
boxed_pixels, box, boxed_points = boxit(image_filename, points=coords) # box also fits now
boxed_pixels = tint(boxed_pixels, 20)
else:
image_filename = "".join((resource_image_path, app.bodypart, ".png"))
boxed_pixels, box, boxed_points = boxit(image_filename, points=coords) # box also fits now
small_pixels, small_size, small_points = smallerize(boxed_pixels, size_factor, box, points=boxed_points)
for act_struct in build_act:
p.update()
angle_list = act_struct.movements[app.bodypart]
for angle in angle_list:
output_name = path.join(output_images, "".join((app.bodypart, "_", str(angle), ".png")))
rotate_pixels, rotate_size, rotate_points = rotate(
small_pixels, app.origin, angle, small_size, points=small_points
)
# save(output_name, small_pixels, small_size)#rotate_pixels, rotate_size)
appendage_files[app].add(angle, dict(parallel(joints, rotate_points)))
for appfile in appendage_files.itervalues():
appfile.push()
t = localtime()
write(complete_file, "%d/%d/%d@%d:%d:%d" % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec))
p.close()
else:
pass
def save_excel(student, cur_dir):
"""
Opens up the relevant excel workbook and logs that the student
has submitted Assignment X at time Y.
"""
if path.exists(str(cur_dir) + '/grive/' + str(student['course']
) + ".xls"):
pass
else:
failure("QR Code Error: Course master spreadsheet " + str(
cur_dir) + '/grive/' + str(student['course']
) + ".xls" + " doesn't exist", cur_dir)
exit()
if path.exists(str(cur_dir) + '/grive/' + str(
student['course']) + "Log.txt"):
log = open(str(cur_dir) + '/grive/' + str(
student['course']) + "Log.txt", "a")
else:
log = open(str(cur_dir) + '/grive/' + str(
student['course']) + "Log.txt", "w")
log.write(str(student['time']) + "\t" + str(student['student number']
) + "\t" + str(student['assignment']) + "\n")
log.close()
book = open_workbook(join(str(cur_dir) + '/grive/' + str(
student['course']) + ".xls"), formatting_info=True, on_demand=True)
worksheet = book.sheet_by_name(str(student['assignment']))
num_rows = worksheet.nrows - 1
# Check submission deadline
student['tutor'] = worksheet.cell_value(0, 1)
deadline = xldate_as_tuple(worksheet.cell_value(1, 1), 0)
# Deadline is automatically set at 3pm on the deadline date
student['deadline'] = datetime(deadline[0], deadline[1],
deadline[2], 18, 0, 0)
work_book = copy(book)
worksheet = get_sheet_by_name(work_book, student['assignment'])
# look through all the student entries, match student number
for i in range(num_rows-2):
if not int(book.sheet_by_name(str(student['assignment'])).cell(
i+3, 1).value) == int(student['student number']):
if i == num_rows-3:
failure("Student " + str(student['student number']) + " not "\
"on course " + str(student['course']) + " master "\
"spreadsheet", cur_dir)
exit()
else:
worksheet.write(i+3, 5, str(student['time']))
if student['time'] > student['deadline']:
worksheet.write(i+3, 6, "Deadline Expired")
break
work_book.save(join(str(cur_dir) + '/grive/' + str(
student['course']) + ".xls"))
return student['deadline']
def main(conf=None):
tabLst = collections.OrderedDict()
tabLst["Mission"] = copy(MISSION)
tabLst["Auditors"] = dict(lst=copy(PEOPLE), db=DBHandler.Auditors(), addFct=addPeople)
tabLst["Clients"] = dict(lst=copy(PEOPLE), db=DBHandler.Clients(), addFct=addPeople)
tabLst["Vulns"] = dict(vulns={"class": Vulns, "arg": (copy(VULNS), DBHandler.Vulns(), addVuln)})
app = QApplication([]) # as we don't use arguments, I prefer to use python-fire
window = Window('Repator', tabLst)
# this allows us to have a directory for every mission with a file that can be automatically configured
# then, just use repator.py --conf=path or repator.py path (or nothing to use default)
if conf is not None:
if path.exists(conf):
config = configparser.ConfigParser()
config.optionxform = str
config.read(conf)
# for a reason I can't explain dates are not parsed... any idea lnv42?
window.loadDict({s: dict(config.items(s)) for s in config.sections()})
else:
window.loadDict({})
# window.loadJson('{"Mission": {"dateStart": "lun. avr. 23 2018", "dateEnd": "ven. avr. 27 2018", "client":"feafe", '
# '"environment": "pr\u00e9-production"}, "Auditors": {}, "Vulns": {"AV1": "P"}}')
window.showMaximized()
app.exec_()
def loadDataframe(folder, ticker):
file_path = getFilePath(folder, ticker + '.csv')
exists = path.exists(file_path)
if exists:
dataframe = pd.read_csv(file_path)
return dataframe
def __init__(self, filename):
TabFile.__init__(self, filename)
self.entries = {}
self.altered = False
if path.exists(self.filename):
self.head()
self.retrieve()
else:
self.header = []
def process_vocab(self, vera_path):
#Accerss all the vera.ai files under the given path and generate the superset - vocab
vocab_path = join(vera_path, 'vocab.vera')
vocab = set([])
for f in glob(path.join(vera_path, '*-vera.ai')):
#for each vera.ai file create a set of words
lines = open(f).readlines()
if len(lines) > 0:
vocab = vocab | set(lines)
if path.exists(vocab_path):
head, tail = path.split(vocab_path)
tail = tail.replace(tail.rfind('.'), '-backup.')
if path.exists(path.join(head, tail)):
os.unlink(path.join(head, tail))
os.rename(vocab_path, path.join(head, tail))
hf = open(vocab_path, 'w')
for item in vocab:
hf.write(item)
hf.close()
def main():
print('== Line Break Tool version 1.5 ==')
# if the config file is not found, exit the program
if not path.exists(CONFIG_FILE):
print(
'Could not find config.json. It must be in the same directory as the program.'
)
sleep(2)
exit('Exiting.')
# if the output directory doesn't exist, try to create it
if not path.exists(OUTPUT_DIR):
print('Output dir does not exist, attempting to create.')
try:
makedirs(OUTPUT_DIR)
print('Successfully created output dir.')
except:
print('Could not create output dir.')
sleep(2)
exit('Exiting.')
lineBreak()
def huffman_check_decompression():
"""
This function allows to determine the source according to the priority
to recover the sequence to be decompressed
"""
global decompress_seq, compress_seq, seq_fasta, binary_sequence, binary_dict, add_zero, button
button = "Decompression"
process_file = ""
dico_string = {}
if seq_fasta != "":
if seq_fasta[0] == "{":
process_file = seq_fasta.split("\n", maxsplit=1)
else:
error("This sequence can not to be decompressed")
elif path.exists("Results/compress_sequence.txt"):
with open("Results/compress_sequence.txt", "r",
encoding="utf-8") as file_compression:
process_file = file_compression.readlines(
) #retrieve all lines from the files
elif path.exists("Results/encrypt_compress_sequence.txt"):
with open("Results/encrypt_compress_sequence.txt",
"r",
encoding="utf-8") as file_encrypt_comp:
process_file = file_encrypt_comp.readlines(
) #retrieve all lines from the files
else:
error("No file for decompression analysis")
if process_file != "":
process_file1 = process_file[0].split("\t")
dico_string = process_file1[0].strip()
add_zero = int(process_file1[1].strip())
compress_seq = "".join(process_file[1:])
if dico_string != {}:
binary_dict = recovery_dict(dico_string)
if compress_seq != "" and add_zero != None and binary_dict != {}:
binary_sequence, decompress_seq = decompression_huffman(
compress_seq, add_zero, binary_dict)
seq_decrypt, list_decrypt, list_decrypt_step = decryptage(
decompress_seq)
def huffman_check_compression():
"""
This function allows to determine the source according to the priorities
to recover the sequence to be compressed
"""
global seq_fasta, origin_seq, compress_seq, button
global given_sequence, binary_sequence, binary_dict, add_zero
button = "Compression"
if sequence.get("1.0", "end-1c") != "":
origin_seq = sequence.get("1.0", "end-1c")
elif seq_fasta != "":
origin_seq = seq_fasta.replace("\n", "")
elif path.exists("Results/encrypt_sequence.txt"):
with open("Results/encrypt_sequence.txt", "r") as file_encrypt:
origin_seq = file_encrypt.readline().strip(
) #retrieve the first line of the file
elif path.exists("Results/decrypt_sequence.txt"):
with open("Results/decrypt_sequence.txt", "r") as file_decrypt:
origin_seq = file_decrypt.readline().strip(
) #retrieve the first line of the file
elif path.exists("Results/decompress_sequence.txt"):
with open("Results/decompress_sequence.txt",
"r") as file_decompression:
origin_seq = file_decompression.readline().strip(
) #retrieve the first line of the file
elif path.exists("Results/decompress_decrypt_sequence.txt"):
with open("Results/decompress_decrypt_sequence.txt",
"r") as file_decomp_decrypt:
origin_seq = file_decomp_decrypt.readline().strip()
else:
error("Please give a study sequence")
#This condition makes it possible to decide whether to launch the analysis or not
if origin_seq != "":
if check_seq(origin_seq.upper()):
given_sequence, binary_dict, compress_seq, add_zero, binary_sequence = compression_huffman(
origin_seq.upper())
else:
error("This sequence can not to be compressed")
def getBackendCreatedFiles ():
from os import path
filesToRemove = []
filesToRemove.append(getMakefileName())
filesToRemove.append(getArchiveFileName())
hiddenFile = None
if path.exists(translatorCUDAFileName) and opts.cuda:
hiddenFile = open(translatorCUDAFileName, 'r')
if path.exists(translatorOpenCLFileName) and opts.opencl:
hiddenFile = open(translatorOpenCLFileName, 'r')
if path.exists(translatorOpenMPFileName) and opts.openmp:
hiddenFile = open(translatorOpenMPFileName, 'r')
generatedCompilationUnit = None
if hiddenFile is not None:
for line in hiddenFile:
line = line.strip()
if line.startswith("generated="):
words = line.split('=')
oldFileName = words[1].strip().split(' ')[0]
newFileName = getBackendSpecificFileName (oldFileName)
generatedCompilationUnit = newFileName
if line.startswith("files="):
words = line.split('=')
files = words[1].strip().split(' ')
for compilationUnitFileName in files:
compilationUnitFileName = compilationUnitFileName.strip()
newFileName = getBackendSpecificFileName(compilationUnitFileName)
if generatedCompilationUnit == newFileName and opts.cuda:
newFileName = getCUDACompilationUnitName(newFileName)
filesToRemove.append(newFileName)
hiddenFile.close()
return filesToRemove
def bwt_check_encryption():
"""
This function allows to determine the source according to the priority
to recover the sequence to be encrypted
"""
global original_seq, list_pattern, list_pattern_sort, seq_fasta, bwt, button
button = "Encryption"
if sequence.get("1.0", "end-1c") != "": #check Text widget
original_seq = sequence.get("1.0", "end-1c")
elif seq_fasta != "": #check open file
original_seq = seq_fasta.strip()
elif path.exists("Results/decrypt_sequence.txt"): #check file exist
with open("Results/decrypt_sequence.txt", "r") as file_decrypt:
original_seq = file_decrypt.readline().strip(
) #retrieve the first line of the file
elif path.exists(
"Results/decompress_sequence.txt"): #check decompress file
with open("Results/decompress_sequence.txt", "r") as file_decomp:
original_seq = file_decomp.readline().strip(
) #retrieve the first line of the file
elif path.exists(
"Results/decompress_decrypt_sequence.txt"): #check file exist
with open("Results/decompress_decrypt_sequence.txt",
"r") as file_decomp_decrypt:
original_seq = file_decomp_decrypt.readline().strip(
) #retrieve the first line of the file
else:
error("Please give a sequence to encrypted")
#This condition makes it possible to decide whether to launch the analysis or not
if original_seq != "":
if not check_dollars(original_seq) and check_seq(
original_seq.upper()): #check format sequence
list_pattern, list_pattern_sort, bwt = cryptage(
original_seq.upper())
else:
error(
"This sequence does not have the right format to be encrypted")
def clean ():
# Cleans out files generated during the compilation process
from glob import glob
from os import path, remove
filesToRemove = []
filesToRemove.extend(glob('*_postprocessed.[fF?]*'))
filesToRemove.extend(glob('*.rmod'))
filesToRemove.extend(glob('*.mod'))
filesToRemove.extend(glob('hs_err_pid*.log'))
filesToRemove.extend(glob('~*'))
filesToRemove.extend(getBackendCreatedFiles())
for file in filesToRemove:
if path.exists(file):
debug.verboseMessage("Removing file: '" + file + "'")
remove(file)
def duo_check_decomp():
"""
This function allows to determine the source according to the priorities
to recover the sequence to be decompressed then decrypted
"""
global seq_fasta, binary_dict, compress_seq, add_zero, binary_sequence
global decompress_seq, seq_decrypt, list_decrypt, list_decrypt_step, button
button = "Decompression/decryption"
process_file = ""
dico_string = {}
if seq_fasta != "":
if seq_fasta[0] == "{":
process_file = seq_fasta.split("\n", maxsplit=1)
else:
error("This sequence can not to be decompressed")
elif path.exists("Results/encrypt_compress_sequence.txt"):
with open("Results/encrypt_compress_sequence.txt",
"r",
encoding="utf-8") as file_duo2:
process_file = file_duo2.readlines(
) #retrieve all lines from the files
else:
error("Please re-encrypt/re_compress")
if process_file != "":
process_file1 = process_file[0].split("\t")
dico_string = process_file1[0].strip()
add_zero = int(process_file1[1].strip())
compress_seq = "".join(
process_file[1:]) #join of the different lines of the file
if dico_string != {}:
binary_dict = recovery_dict(dico_string)
#This condition makes it possible to decide whether to launch the analysis or not
if compress_seq != "" and add_zero != None and binary_dict != {}:
binary_sequence, decompress_seq = decompression_huffman(
compress_seq, add_zero, binary_dict)
if check_dollars(decompress_seq):
seq_decrypt, list_decrypt, list_decrypt_step = decryptage(
decompress_seq)
else:
error("The file is not encrypt")
redirect_path = apache2_sites_path + "redirect/"
# endregion
# region Set phishing domain and path
se_domain = args.phishing_domain
if args.phishing_domain_path == "google" or "apple" or "microsoft":
se_path = apache2_sites_path + args.phishing_domain_path
else:
se_path = args.phishing_domain_path
Base.print_info("Phishing domain: ", se_domain)
Base.print_info("Phishing domain local path: ", se_path)
# endregion
# region Directory for phishing domain
if not path.exists(se_path):
if args.phishing_domain_path == "google" or "apple" or "microsoft":
copytree(src=project_root_path +
"/raw_packet/Utils/Phishing_domains/" +
args.phishing_domain_path,
dst=se_path)
else:
Base.print_error("Directory: ", se_path, " does not exist!")
exit(1)
credentials_file_name = se_path + '/logins.txt'
sub.Popen(['chmod 777 ' + credentials_file_name + ' >/dev/null 2>&1'],
shell=True)
# endregion
# region Apache2 sites settings
# -*- coding: utf-8 -*-
import numpy as np
from matplotlib import pyplot as plt
from sklearn import datasets
from sys import path as path
from os import path, makedirs
from scipy.spatial.distance import pdist
import myDA as da
from sklearn.utils import shuffle
from sklearn.metrics.pairwise import rbf_kernel
from madaot import Madaot
from sklearn.metrics import accuracy_score
resultsPath = "results/DA/"
if not path.exists(resultsPath):
makedirs(resultsPath)
import pickle as pkl
plt.rcParams.update({
"font.family":
"serif",
"pgf.texsystem":
"pdflatex",
"pgf.preamble": [
r"\usepackage[utf8]{inputenc}",
r"\usepackage[T1]{fontenc}",
r"\usepackage{serif}",
r"\usepackage{amsmath, amsfonts, amssymb, amstext, amsthm, bbm, mathtools}",
]
y = 0
while y != 1:
print('Please mention the complete path of the image to fetch the file ')
ip_path = input(f'Path:\t')
pattern = re.compile('^[A-Z][:]{1}\\\(.*)')
match = pattern.match(ip_path)
if match:
y = 1
else:
print('Please enter the correct path')
time.sleep(3)
k = 0
while k != 1:
file_name = input(f'Please enter the File Name to be opened:\t')
image_path = fr'{ip_path}' f'\\{file_name}'
img_exist = path.exists(fr'{image_path}')
if img_exist:
k = 1
else:
print('Entered file doesnot exist. Please check and try again')
time.sleep(3)
exit_id: str = 'No'
while 'Yes' != exit_id:
print("-" * 100)
print("*" * 10 + " Image Processing Operations " + "*" * 10)
print(
f'1 : Create Thumbnail \n 2 : Convert Image mode \n 3 : Image Enhancement \n 4 : Convert Image Format \n 5 : Image Compression '
)
print("-" * 100)
data_input = input(f'Please enter the Choice : \t')
if data_input == '1':
def existsFile(folder, ticker):
file_path = getFilePath(folder, ticker + '.csv')
exists = path.exists(file_path)
return exists
def load_file(self) -> dict:
if not path.exists(self.filename):
return {}
with open(self.filename, 'r') as f:
return JsonWrapper.load(f)
def parse_stats_from_runs(experiment_name, gen_no=1000, skip=False):
"""
Analyses a list of given stats from a group of runs saved under an
"experiment_name" folder. Creates a summary .csv file which can be used by
plotting functions in utilities.save_plot. Saves a file of the format:
run0_gen0 run1_gen0 . . . run(n-1)_gen0
run0_gen1 run1_gen1 . . . run(n-1)_gen1
run0_gen2 run1_gen2 . . . run(n-1)_gen2
. . . . . .
. . . . . .
. . . . . .
run0_gen(n-1) run1_gen(n-1) . . . run(n-1)_gen(n-1)
run0_gen(n) run1_gen(n) . . . run(n-1)_gen(n)
Generated file is compatible with
utilities.save_plot.save_average_plot_across_runs()
:param experiment_name: The name of a collecting folder within the
./results folder which holds multiple runs.
:param graph: A boolean flag for whether or not to save figure.
:return: Nothing.
"""
# Since results files are not kept in source directory, need to escape
# one folder.
file_path = path.join(getcwd(), "..", "results")
# Check for use of experiment manager.
if experiment_name:
file_path = path.join(file_path, experiment_name)
else:
s = "scripts.parse_stats.parse_stats_from_runs\n" \
"Error: experiment name not specified."
raise Exception(s)
# Find list of all runs contained in the specified folder.
runs = [
run for run in listdir(file_path)
if path.isdir(path.join(file_path, run))
]
# Place to store the header for full stats file.
header = ""
# Array to store all stats
full_stats = []
# Get list of all stats to parse. Check stats file of first run from
# runs folder.
ping_file = path.join(file_path, str(runs[0]), "stats.tsv")
# Load in data and get the names of all stats.
stats = list(pd.read_csv(ping_file, sep="\t"))
# Make list of stats we do not wish to parse.
no_parse_list = ["gen", "total_inds", "time_adjust", "best_phenotype"]
gen_num = int(gen_no)
if skip:
gen_no += "S2"
if not path.exists(path.join(file_path + "-results", gen_no)):
makedirs(path.join(file_path + "-results", gen_no))
for stat in [
stat for stat in stats
if stat not in no_parse_list and not stat.startswith("Unnamed")
]:
# Iterate over all stats.
print("Parsing", stat)
summary_stats = []
# Iterate over all runs
for run in runs:
# Get file name
file_name = path.join(file_path, str(run), "stats.tsv")
# Load in data
data = pd.read_csv(file_name, sep="\t")
try:
# Try to extract specific stat from the data.
if list(data[stat]):
data_local = list(data[stat])
if len(data_local):
if skip and len(data_local) < 3 / 4 * gen_num:
continue
for i in range(len(data_local), gen_num):
data_local.append(data_local[-1])
summary_stats.append(data_local[:gen_num])
else:
s = "scripts.parse_stats.parse_stats_from_runs\n" \
"Error: stat %s is empty for run %s." % (stat, run)
raise Exception(s)
except KeyError:
# The requested stat doesn't exist.
s = "scripts.parse_stats.parse_stats_from_runs\nError: " \
"stat %s does not exist in run %s." % (stat, run)
raise Exception(s)
try:
# Generate numpy array of all stats
summary_stats = np.array(summary_stats)
# Append Stat to header.
header = header + stat + "_mean,"
summary_stats_mean = np.nanmean(summary_stats, axis=0)
full_stats.append(summary_stats_mean)
# Append Stat to header.
header = header + stat + "_std,"
summary_stats_std = np.nanstd(summary_stats, axis=0)
full_stats.append(summary_stats_std)
summary_stats = np.transpose(summary_stats)
# Save stats as a .csv file.
np.savetxt(path.join(file_path + "-results", str(gen_no),
(stat + ".csv")),
summary_stats,
delimiter=",")
# Graph stat by calling graphing function.
save_average_plot_across_runs(
path.join(file_path + "-results", str(gen_no),
(stat + ".csv")))
except FloatingPointError:
print("scripts.stats_parser.parse_stats_from_runs\n"
"Warning: FloatingPointError encountered while parsing %s "
"stats." % (stat))
stat = 'metrics_average'
print("Parsing", stat)
summary_stats = []
for run in runs:
# Get file name
file_name = path.join(file_path, str(run), "stats.tsv")
# Load in data
data = pd.read_csv(file_name, sep="\t")
try:
alignment = list(data['alignment'])
if alignment:
if len(alignment) < gen_num:
print(len(alignment))
if skip and len(data_local) < 3 / 4 * gen_num:
continue
for i in range(len(alignment), gen_num):
alignment.append(alignment[-1])
generalization = list(data['generalization'])
if generalization:
if len(generalization) < gen_num:
if skip and len(data_local) < 3 / 4 * gen_num:
continue
for i in range(len(generalization), gen_num):
generalization.append(generalization[-1])
precision = list(data['precision'])
if precision:
if len(precision) < gen_num:
if skip and len(data_local) < 3 / 4 * gen_num:
continue
for i in range(len(precision), gen_num):
precision.append(precision[-1])
simplicity = list(data['simplicity'])
if simplicity:
if len(simplicity) < gen_num:
if skip and len(data_local) < 3 / 4 * gen_num:
continue
for i in range(len(simplicity), gen_num):
simplicity.append(simplicity[-1])
data_local = []
for i in range(gen_num):
data_local.append((8 * alignment[i] + 2 * generalization[i] +
4 * precision[i] + 2 * simplicity[i]) / 16)
summary_stats.append(data_local[:gen_num])
except KeyError:
# The requested stat doesn't exist.
s = "scripts.parse_stats.parse_stats_from_runs\nError: " \
"stat %s does not exist in run %s." % (stat, run)
raise Exception(s)
try:
# Generate numpy array of all stats
summary_stats = np.array(summary_stats)
# Append Stat to header.
header = header + stat + "_mean,"
summary_stats_mean = np.nanmean(summary_stats, axis=0)
full_stats.append(summary_stats_mean)
# Append Stat to header.
header = header + stat + "_std,"
summary_stats_std = np.nanstd(summary_stats, axis=0)
full_stats.append(summary_stats_std)
summary_stats = np.transpose(summary_stats)
# Save stats as a .csv file.
np.savetxt(path.join(file_path + "-results", str(gen_no),
(stat + ".csv")),
summary_stats,
delimiter=",")
# Graph stat by calling graphing function.
save_average_plot_across_runs(
path.join(file_path + "-results", str(gen_no), (stat + ".csv")))
except FloatingPointError:
print("scripts.stats_parser.parse_stats_from_runs\n"
"Warning: FloatingPointError encountered while parsing %s "
"stats." % (stat))
# Convert and rotate full stats
full_stats = np.array(full_stats)
full_stats = np.transpose(full_stats)
# Save full stats to csv file.
np.savetxt(path.join(file_path + "-results", str(gen_no),
"full_stats.csv"),
full_stats,
delimiter=",",
header=header[:-1])
parser.add_option('--weights', dest="weights", default="")
parser.add_option('--limitFile', dest="limitFile", default="")
parser.add_option('--noSyst',
dest="doSyst",
action="store_false",
default=True)
parser.add_option('--nameSuffix', dest="histNameSuffix", default="")
o, a = parser.parse_args()
###########
## Setup ##
###########
if o.year == "2016": lumi = "24.3 fb^{-1}"
if path.exists(o.limitFile):
limitFile = TFile.Open(o.limitFile, "UPDATE")
else:
limitFile = TFile.Open(o.limitFile, "RECREATE")
inDir = "hists_" + o.year + "_" + o.weights
files = rootFiles.getFiles(o.iteration, o.nTuple, inDir, o.year)
shapeVariation = "_CR"
#shapeVariation="_CR"
if o.doSyst:
filesCRw = rootFiles.getFiles(o.iteration, o.nTuple,
inDir + shapeVariation, o.year)
muFile = "XhhResolved/data/mu_qcd_FourTag_" + o.weights + "_" + o.year + "_" + o.variation + "_" + o.iteration + ".txt"
mu_qcd_dict = read_mu_qcd_file(muFile)
if o.doSyst:
def lineBreak():
# open the config file
config = load(open(CONFIG_FILE, encoding='utf_8'))
try:
fileDirectory = config['fileDirectory']
# make sure the directory is valid
if not path.exists(fileDirectory):
if fileDirectory == '':
fileDirectory = '.'
else:
print(
'Could not find the directory \"{}\", please check to make sure it is correct.'
.format(fileDirectory))
sleep(2)
exit('Exiting.')
except:
print('Could not find the file directory defined in config.json')
sleep(2)
exit('Exiting.')
# load values from config.json
try:
charMax = int(config['maxCharacters'])
breakCharacter = config['breakCharacter']
fileTypes = config['fileTypes'].split(', ')
# if no file types are defined, exit the program
if fileTypes == ['']:
print('No file types have been defined in config.json.')
sleep(2)
exit('Exiting.')
selectedType = config['chosenType']
skipCountCharacters = config['skipCountCharacters'].split(', ')
if skipCountCharacters == ['']:
skipCountCharacters = ''
setEncoding = config['encoding']
cleanupEnabled = config['enableCleanup']
lineTypeIgnoreJson = config[selectedType]
lineTypeIgnoreList = []
for key, value in lineTypeIgnoreJson.items():
lineTypeIgnoreList.append(value)
except:
print(
'Some values in config.json can not be found or are entered incorrectly.'
)
sleep(2)
exit('Exiting')
for file in listdir(fileDirectory):
if file.endswith(tuple(fileTypes)):
# check and handle different encodings
try:
openFile = open(path.join(fileDirectory, file),
'r',
encoding=setEncoding)
except:
try:
openFile = open(path.join(fileDirectory, file),
'r',
encoding='utf_8')
print('Opened the file in utf-8 instead of {}.'.format(
setEncoding))
except UnicodeDecodeError:
print(
'Could not decode file \"{}\", set a different encoding in config.json'
.format(file))
continue
readFile = openFile.readlines()
outputFile = open(OUTPUT_DIR + file, 'w', encoding=setEncoding)
for line in readFile:
if any(x in line for x in lineTypeIgnoreList):
outputFile.write(line)
else:
# if cleanup is enabled in the config
if cleanupEnabled == 'true':
# remove existing break characters
line = line.replace(breakCharacter, '')
# replace double spaces with single spaces
line = line.replace(' ', ' ')
# replace those weird ellipses'
line = line.replace('…', '...')
# split the string into a list of words
wordList = line.split(' ')
characterCount = 0
finishedLine = ''
for word in wordList:
# count the characters in the word, add one for the space that was taken out
characterCount += len(word) + 1
# if the word has characters than are set to be skipped, subtract them from the count
if any(x in word for x in skipCountCharacters):
for value in skipCountCharacters:
if value in word:
subtract = len(value)
characterCount -= subtract
# if the character count exceeds the maximum allowed
if characterCount > charMax:
finishedLine += breakCharacter
characterCount = 0
characterCount += len(word) + 1
finishedLine += word
continue
finishedLine += ' ' + word
# remove the space at the beginning of the string
outputFile.write(finishedLine[1:])
openFile.close()
outputFile.close()
print('Successfully added line breaks to \"{}\".'.format(file))
else:
print('File \"{}\" was skipped over.'.format(file))
continue
