def main_from_dir(path):
log_file = os.path.join(path, "log.csv")
path_exists = os.path.exists(path)
log_file_exists = os.path.exists(log_file)
examples_from_folder = isinstance(app.interface.examples, str) and app.interface.examples == path
multiple_inputs = len(app.interface.input_interfaces) > 1
if path_exists:
if not log_file_exists and multiple_inputs:
if examples_from_folder:
abort(404, "log.csv file required for multiple inputs.")
else:
return redirect("/")
elif examples_from_folder:
abort(404, "Examples dir not found")
else:
return redirect("/")
if log_file_exists:
if app.interface.encrypt:
with open(log_file, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(
app.interface.encryption_key, encrypted_csv)
csv_data = io.StringIO(decrypted_csv.decode())
examples = list(csv.reader(csv_data))
else:
with open(log_file) as logs:
examples = list(csv.reader(logs))
examples = examples[1:] # remove header
else:
examples = [[filename] for filename in os.listdir(path)]
for i, example in enumerate(examples):
for j, (interface, cell) in enumerate(zip(app.interface.input_interfaces + app.interface.output_interfaces, example)):
examples[i][j] = interface.restore_flagged(cell)
return home_page(examples=examples, path=path)
def main_from_dir(path):
log_file = os.path.join(path, "log.csv")
if not os.path.exists(log_file):
if isinstance(app.interface.examples, str):
abort(404, "Examples dir not found")
else:
redirect("/")
if app.interface.encrypt:
with open(log_file, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(app.interface.encryption_key,
encrypted_csv)
csv_data = io.StringIO(decrypted_csv.decode())
examples = list(csv.reader(csv_data))
else:
with open(log_file) as logs:
examples = list(csv.reader(logs))
examples = examples[1:] #remove header
for i, example in enumerate(examples):
for j, (interface, cell) in enumerate(
zip(
app.interface.input_interfaces +
app.interface.output_interfaces, example)):
examples[i][j] = interface.restore_flagged(cell)
return home_page(examples=examples, path=path)
def file(path):
if app.interface.encrypt and isinstance(app.interface.examples, str) and path.startswith(app.interface.examples):
with open(os.path.join(app.cwd, path), "rb") as encrypted_file:
encrypted_data = encrypted_file.read()
file_data = encryptor.decrypt(
app.interface.encryption_key, encrypted_data)
return send_file(io.BytesIO(file_data), attachment_filename=os.path.basename(path))
else:
return send_file(os.path.join(app.cwd, path))
def encode_file_to_base64(f, encryption_key=None):
with open(f, "rb") as file:
encoded_string = base64.b64encode(file.read())
if encryption_key:
encoded_string = encryptor.decrypt(encryption_key, encoded_string)
base64_str = str(encoded_string, 'utf-8')
mimetype = get_mimetype(f)
return "data:" + (mimetype if mimetype is not None else
"") + ";base64," + base64_str
def flag_data(input_data, output_data, flag_option=None):
flag_path = os.path.join(app.cwd, app.interface.flagging_dir)
csv_data = []
encryption_key = app.interface.encryption_key if app.interface.encrypt else None
for i, interface in enumerate(app.interface.input_interfaces):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["input_interfaces"][i][1]["label"],
input_data[i], encryption_key))
for i, interface in enumerate(app.interface.output_interfaces):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["output_interfaces"][i][1]["label"],
output_data[i], encryption_key))
if flag_option:
csv_data.append(flag_option)
headers = [
interface[1]["label"]
for interface in app.interface.config["input_interfaces"]
]
headers += [
interface[1]["label"]
for interface in app.interface.config["output_interfaces"]
]
if app.interface.flagging_options is not None:
headers.append("flag")
log_fp = "{}/log.csv".format(flag_path)
is_new = not os.path.exists(log_fp)
if app.interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(app.interface.encryption_key,
encrypted_csv)
output.write(decrypted_csv.decode())
writer = csv.writer(output)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(app.interface.encryption_key,
output.getvalue().encode()))
else:
with open(log_fp, "a") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
def file(path):
if (app.blocks.encrypt and isinstance(app.blocks.examples, str)
and path.startswith(app.blocks.examples)):
with open(safe_join(app.cwd, path), "rb") as encrypted_file:
encrypted_data = encrypted_file.read()
file_data = encryptor.decrypt(app.blocks.encryption_key,
encrypted_data)
return FileResponse(io.BytesIO(file_data),
attachment_filename=os.path.basename(path))
else:
if Path(app.cwd).resolve() in Path(path).resolve().parents:
return FileResponse(Path(path).resolve())
def flag_data(input_data,
output_data,
flag_option=None,
flag_index=None,
username=None):
flag_path = os.path.join(app.cwd, app.interface.flagging_dir)
log_fp = "{}/log.csv".format(flag_path)
encryption_key = app.interface.encryption_key if app.interface.encrypt else None
is_new = not os.path.exists(log_fp)
if flag_index is None:
csv_data = []
for i, interface in enumerate(app.interface.input_components):
csv_data.append(
interface.save_flagged(
flag_path,
app.interface.config["input_components"][i]["label"],
input_data[i], encryption_key))
for i, interface in enumerate(app.interface.output_components):
csv_data.append(
interface.save_flagged(
flag_path, app.interface.config["output_components"][i]
["label"], output_data[i], encryption_key
) if output_data[i] is not None else "")
if flag_option is not None:
csv_data.append(flag_option)
if username is not None:
csv_data.append(username)
if is_new:
headers = [
interface["label"]
for interface in app.interface.config["input_components"]
]
headers += [
interface["label"]
for interface in app.interface.config["output_components"]
]
if app.interface.flagging_options is not None:
headers.append("flag")
if username is not None:
headers.append("username")
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output)
writer.writerows(content)
return output.getvalue()
if app.interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(app.interface.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output)
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(app.interface.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_fp, "a") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_fp) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_fp, "w") as csvfile:
csvfile.write(file_content)
with open(log_fp, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count
def flag(
self,
flag_data: List[Any],
flag_option: Optional[str] = None,
flag_index: Optional[int] = None,
username: Optional[str] = None,
) -> int:
flagging_dir = self.flagging_dir
log_filepath = os.path.join(flagging_dir, "log.csv")
is_new = not os.path.exists(log_filepath)
if flag_index is None:
csv_data = []
for component, sample in zip(self.components, flag_data):
csv_data.append(
component.save_flagged(
flagging_dir,
component.label,
sample,
self.encryption_key,
) if sample is not None else "")
csv_data.append(flag_option if flag_option is not None else "")
csv_data.append(username if username is not None else "")
csv_data.append(str(datetime.datetime.now()))
if is_new:
headers = [component.label
for component in self.components] + [
"flag",
"username",
"timestamp",
]
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
writer.writerows(content)
return output.getvalue()
if self.encryption_key:
output = io.StringIO()
if not is_new:
with open(log_filepath, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(self.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_filepath, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(self.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_filepath, "a", newline="") as csvfile:
writer = csv.writer(csvfile,
quoting=csv.QUOTE_NONNUMERIC,
quotechar="'")
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_filepath) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_filepath, "w", newline=""
) as csvfile: # newline parameter needed for Windows
csvfile.write(file_content)
with open(log_filepath, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count
def test_same_pass(self):
key = encryptor.get_key("test")
data, _ = processing_utils.decode_base64_to_binary(BASE64_IMAGE)
encrypted_data = encryptor.encrypt(key, data)
decrypted_data = encryptor.decrypt(key, encrypted_data)
self.assertEqual(data, decrypted_data)
def flag(self,
interface,
input_data,
output_data,
flag_option=None,
flag_index=None,
username=None):
flagging_dir = self.flagging_dir
log_fp = "{}/log.csv".format(flagging_dir)
encryption_key = interface.encryption_key if interface.encrypt else None
is_new = not os.path.exists(log_fp)
output_only_mode = input_data is None
if flag_index is None:
csv_data = []
if not output_only_mode:
for i, input in enumerate(interface.input_components):
csv_data.append(
input.save_flagged(
flagging_dir,
interface.config["input_components"][i]["label"],
input_data[i], encryption_key))
for i, output in enumerate(interface.output_components):
csv_data.append(
output.save_flagged(
flagging_dir, interface.config["output_components"][i]
["label"], output_data[i], encryption_key
) if output_data[i] is not None else "")
if not output_only_mode:
if flag_option is not None:
csv_data.append(flag_option)
if username is not None:
csv_data.append(username)
csv_data.append(str(datetime.datetime.now()))
if is_new:
headers = []
if not output_only_mode:
headers += [
interface["label"]
for interface in interface.config["input_components"]
]
headers += [
interface["label"]
for interface in interface.config["output_components"]
]
if not output_only_mode:
if interface.flagging_options is not None:
headers.append("flag")
if username is not None:
headers.append("username")
headers.append("timestamp")
def replace_flag_at_index(file_content):
file_content = io.StringIO(file_content)
content = list(csv.reader(file_content))
header = content[0]
flag_col_index = header.index("flag")
content[flag_index][flag_col_index] = flag_option
output = io.StringIO()
writer = csv.writer(output)
writer.writerows(content)
return output.getvalue()
if interface.encrypt:
output = io.StringIO()
if not is_new:
with open(log_fp, "rb") as csvfile:
encrypted_csv = csvfile.read()
decrypted_csv = encryptor.decrypt(interface.encryption_key,
encrypted_csv)
file_content = decrypted_csv.decode()
if flag_index is not None:
file_content = replace_flag_at_index(file_content)
output.write(file_content)
writer = csv.writer(output)
if flag_index is None:
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
with open(log_fp, "wb") as csvfile:
csvfile.write(
encryptor.encrypt(interface.encryption_key,
output.getvalue().encode()))
else:
if flag_index is None:
with open(log_fp, "a", newline="") as csvfile:
writer = csv.writer(csvfile)
if is_new:
writer.writerow(headers)
writer.writerow(csv_data)
else:
with open(log_fp) as csvfile:
file_content = csvfile.read()
file_content = replace_flag_at_index(file_content)
with open(log_fp, "w", newline=""
) as csvfile: # newline parameter needed for Windows
csvfile.write(file_content)
with open(log_fp, "r") as csvfile:
line_count = len([None for row in csv.reader(csvfile)]) - 1
return line_count