def CreateFinalForm(input: Path, output: Path):
inputs = [ input ]
io.msg(f"converting into final format output {output} from files {inputs}")
dataframe = flist.MCSV_Dataset.Dataframe_From_Files(inputs)
mcsv_set = flist.MCSV_Dataset.From_Dataframe(dataframe)
finalForm = flist.FinalForm_Dataset.From_MCSV(mcsv_set)
finalForm.write_csv(output)
def write_csv(self, outfile):
df = self.to_dataframe()
df.insert(0, "ID", range(len(self.rows)))
df.to_csv(outfile,
quoting=csv.QUOTE_NONE,
sep=CSV_SEP,
index=False,
header=False)
io.msg(f"written CSV output to {outfile}")
def refresh_workspace():
if workspace.exists():
shutil.rmtree(workspace)
os.makedirs(workspace)
io.msg(f"populating workspace: {workspace}")
for subelement in ws_static_content_dir.glob("*"):
shutil.copytree(
subelement,
workspace / subelement.relative_to(ws_static_content_dir))
def write_csv(self, outfile):
"""
This attempts to convert the CSV_Entry fields from self.get_dataframe_dictionaries to a pandas.DataFrame,
which is then written to a CSV file.
"""
self.to_dataframe().to_csv(outfile,
quoting=csv.QUOTE_NONE,
sep=CSV_SEP,
index=False,
header=False)
io.msg(f"written CSV output to {outfile}")
def PreprocessCT2(input: Path, output: Path, language: str):
"""
preprocess csv generated by CT2 according to the findings in doc/ct2-generated-csv/incongruence.md
"""
# read in
if not input or not input.exists():
raise io.FlistException(f"input file {input} does not exist")
if input == output:
raise io.FlistException(f"{input=} cannot be equal to {output=}")
if language == "en":
# delete:
# 1a)
# SIGABA Known Plaintext;C;
# ;[C];Tools\\ Misc\\ SIGABA Known Plaintext
# 1b)
# Ciphertext-only;W;
# [...]
# ;[W];Cryptanalysis\\ Modern Encryption\\ Symmetric Encryption\\ DES\\ Ciphertext-only
with open(input, "r") as inputreader:
lines = inputreader.readlines()
with open(output, "w") as outputwriter:
encountered1a = False
encountered1b = False
fixed1a = None
fixed1b = None
for line in lines:
# Filter 1a) block
if "SIGABA Known Plaintext;C;" in line:
encountered1a = True
continue
if encountered1a and len(line.strip()) == 0:
encountered1a = False
# outputwriter.write(line)
continue
if encountered1a:
fixed1a = True
continue
# Filter 1a) block
if "Ciphertext-only;W;" in line:
encountered1b = True
outputwriter.write(line)
continue
if encountered1b and len(line.strip()) == 0:
encountered1b = False
outputwriter.write(line)
continue
if encountered1b and ";[W];Cryptanalysis\\ Modern Encryption\\ Symmetric Encryption\\ DES\\ Ciphertext-only" in line:
fixed1b = True
continue
if encountered1b:
outputwriter.write(line)
continue
outputwriter.write(line)
if fixed1a is None:
io.msg(
f"[[ WARNING ]] when preprocessing CT2 files, did not encounter special case 1a as described in doc/ct2-generated-csv/incongruence.md . This may be a non-issue, though."
)
if fixed1b is None:
io.msg(
f"[[ WARNING ]] when preprocessing CT2 files, did not encounter special case 1b as described in doc/ct2-generated-csv/incongruence.md . This may be a non-issue, though."
)
elif language == "de":
with open(input, "r") as inputreader:
lines = inputreader.readlines()
with open(output, "w") as outputwriter:
encountered2a = None
fixed2a = None
for line in lines:
# Filter 2a) block
if "Ciphertext-only-Analyse;W;" in line:
encountered2a = True
continue
if encountered2a and len(line.strip()) == 0:
encountered2a = False
# outputwriter.write(line)
continue
if encountered2a:
fixed2a = True
continue
outputwriter.write(line)
if fixed2a is None:
io.msg(
f"[[ WARNING ]] when preprocessing CT2 files, did not encounter special case 2a as described in doc/ct2-generated-csv/incongruence.md . This may be a non-issue, though."
)
# 2a)
# Ciphertext-only-Analyse;W;
# ;[W];Kryptoanalyse\\ Moderne Verschlüsselung\\ Symmetrische Verschlüsselung\\ DES\\ Ciphertext-only-Analyse
else:
raise io.FlistException(f"unknown {language=}")
# write out
return