def flag_data(input_data, output_data, flag_option=None):
flag_path = os.path.join(app.cwd, app.interface.flagging_dir)
csv_data = []
encryption_key = app.interface.encryption_key if app.interface.encrypt else None
for i, interface in enumerate(app.interface.input_interfaces):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["input_interfaces"][i][1]["label"],
input_data[i], encryption_key))
for i, interface in enumerate(app.interface.output_interfaces):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["output_interfaces"][i][1]["label"],
output_data[i], encryption_key))
if flag_option:
csv_data.append(flag_option)
headers = [
interface[1]["label"]
for interface in app.interface.config["input_interfaces"]
]
headers += [
interface[1]["label"]
for interface in app.interface.config["output_interfaces"]
]
if app.interface.flagging_options is not None:
headers.append("flag")
log_fp = "{}/log.csv".format(flag_path)
is_new = not os.path.exists(log_fp)
if app.interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(app.interface.encryption_key,
encrypted_csv)
output.write(decrypted_csv.decode())
writer = csv.writer(output)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(app.interface.encryption_key,
output.getvalue().encode()))
else:
with open(log_fp, "a") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
def decode_base64_to_file(encoding, encryption_key=None):
data, extension = decode_base64_to_binary(encoding)
if extension is None:
file_obj = tempfile.NamedTemporaryFile(delete=False)
else:
file_obj = tempfile.NamedTemporaryFile(delete=False, suffix="."+extension)
if encryption_key is not None:
data = encryptor.encrypt(encryption_key, data)
file_obj.write(data)
file_obj.flush()
return file_obj
def decode_base64_to_file(encoding, encryption_key=None, filename_prefix=""):
data, extension = decode_base64_to_binary(encoding)
if extension is None:
file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=filename_prefix)
else:
file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=filename_prefix, suffix="."+extension)
if encryption_key is not None:
data = encryptor.encrypt(encryption_key, data)
#print("saving to ", file_obj.name)
file_obj.write(data)
file_obj.flush()
return file_obj
def decode_base64_to_file(encoding, encryption_key=None, file_path=None):
data, extension = decode_base64_to_binary(encoding)
prefix = None
if file_path is not None:
filename = os.path.basename(file_path)
prefix = filename
if "." in filename:
prefix = filename[0 : filename.index(".")]
extension = filename[filename.index(".") + 1 :]
if extension is None:
file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=prefix)
else:
file_obj = tempfile.NamedTemporaryFile(
delete=False, prefix=prefix, suffix="." + extension
)
if encryption_key is not None:
data = encryptor.encrypt(encryption_key, data)
file_obj.write(data)
file_obj.flush()
return file_obj
def flag_data(input_data,
output_data,
flag_option=None,
flag_index=None,
username=None):
flag_path = os.path.join(app.cwd, app.interface.flagging_dir)
log_fp = "{}/log.csv".format(flag_path)
encryption_key = app.interface.encryption_key if app.interface.encrypt else None
is_new = not os.path.exists(log_fp)
if flag_index is None:
csv_data = []
for i, interface in enumerate(app.interface.input_components):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["input_components"][i]["label"],
input_data[i], encryption_key))
for i, interface in enumerate(app.interface.output_components):
csv_data.append(
interface.save_flagged(
flag_path, app.interface.config["output_components"][i]
["label"], output_data[i], encryption_key
) if output_data[i] is not None else "")
if flag_option is not None:
csv_data.append(flag_option)
if username is not None:
csv_data.append(username)
if is_new:
headers = [
interface["label"]
for interface in app.interface.config["input_components"]
]
headers += [
interface["label"]
for interface in app.interface.config["output_components"]
]
if app.interface.flagging_options is not None:
headers.append("flag")
if username is not None:
headers.append("username")
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output)
writer.writerows(content)
return output.getvalue()
if app.interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(app.interface.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output)
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(app.interface.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_fp, "a") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_fp) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_fp, "w") as csvfile:
csvfile.write(file_content)
with open(log_fp, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count
def flag(
self,
flag_data: List[Any],
flag_option: Optional[str] = None,
flag_index: Optional[int] = None,
username: Optional[str] = None,
) -> int:
flagging_dir = self.flagging_dir
log_filepath = os.path.join(flagging_dir, "log.csv")
is_new = not os.path.exists(log_filepath)
if flag_index is None:
csv_data = []
for component, sample in zip(self.components, flag_data):
csv_data.append(
component.save_flagged(
flagging_dir,
component.label,
sample,
self.encryption_key,
) if sample is not None else "")
csv_data.append(flag_option if flag_option is not None else "")
csv_data.append(username if username is not None else "")
csv_data.append(str(datetime.datetime.now()))
if is_new:
headers = [component.label
for component in self.components] + [
"flag",
"username",
"timestamp",
]
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
writer.writerows(content)
return output.getvalue()
if self.encryption_key:
output = io.StringIO()
if not is_new:
with open(log_filepath, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(self.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_filepath, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(self.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_filepath, "a", newline="") as csvfile:
writer = csv.writer(csvfile,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_filepath) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_filepath, "w", newline=""
) as csvfile: # newline parameter needed for Windows
csvfile.write(file_content)
with open(log_filepath, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count
def test_same_pass(self):
key = encryptor.get_key("test")
data, _ = processing_utils.decode_base64_to_binary(BASE64_IMAGE)
encrypted_data = encryptor.encrypt(key, data)
decrypted_data = encryptor.decrypt(key, encrypted_data)
self.assertEqual(data, decrypted_data)
def flag(self,
interface,
input_data,
output_data,
flag_option=None,
flag_index=None,
username=None):
flagging_dir = self.flagging_dir
log_fp = "{}/log.csv".format(flagging_dir)
encryption_key = interface.encryption_key if interface.encrypt else None
is_new = not os.path.exists(log_fp)
output_only_mode = input_data is None
if flag_index is None:
csv_data = []
if not output_only_mode:
for i, input in enumerate(interface.input_components):
csv_data.append(
input.save_flagged(
flagging_dir,
interface.config["input_components"][i]["label"],
input_data[i], encryption_key))
for i, output in enumerate(interface.output_components):
csv_data.append(
output.save_flagged(
flagging_dir, interface.config["output_components"][i]
["label"], output_data[i], encryption_key
) if output_data[i] is not None else "")
if not output_only_mode:
if flag_option is not None:
csv_data.append(flag_option)
if username is not None:
csv_data.append(username)
csv_data.append(str(datetime.datetime.now()))
if is_new:
headers = []
if not output_only_mode:
headers += [
interface["label"]
for interface in interface.config["input_components"]
]
headers += [
interface["label"]
for interface in interface.config["output_components"]
]
if not output_only_mode:
if interface.flagging_options is not None:
headers.append("flag")
if username is not None:
headers.append("username")
headers.append("timestamp")
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output)
writer.writerows(content)
return output.getvalue()
if interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(interface.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output)
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(interface.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_fp, "a", newline="") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_fp) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_fp, "w", newline=""
) as csvfile: # newline parameter needed for Windows
csvfile.write(file_content)
with open(log_fp, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count
