def main_from_dir(path): log_file = os.path.join(path, "log.csv") path_exists = os.path.exists(path) log_file_exists = os.path.exists(log_file) examples_from_folder = isinstance(app.interface.examples, str) and app.interface.examples == path multiple_inputs = len(app.interface.input_interfaces) > 1 if path_exists: if not log_file_exists and multiple_inputs: if examples_from_folder: abort(404, "log.csv file required for multiple inputs.") else: return redirect("/") elif examples_from_folder: abort(404, "Examples dir not found") else: return redirect("/") if log_file_exists: if app.interface.encrypt: with open(log_file, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt( app.interface.encryption_key, encrypted_csv) csv_data = io.StringIO(decrypted_csv.decode()) examples = list(csv.reader(csv_data)) else: with open(log_file) as logs: examples = list(csv.reader(logs)) examples = examples[1:] # remove header else: examples = [[filename] for filename in os.listdir(path)] for i, example in enumerate(examples): for j, (interface, cell) in enumerate(zip(app.interface.input_interfaces + app.interface.output_interfaces, example)): examples[i][j] = interface.restore_flagged(cell) return home_page(examples=examples, path=path)
def main_from_dir(path): log_file = os.path.join(path, "log.csv") if not os.path.exists(log_file): if isinstance(app.interface.examples, str): abort(404, "Examples dir not found") else: redirect("/") if app.interface.encrypt: with open(log_file, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(app.interface.encryption_key, encrypted_csv) csv_data = io.StringIO(decrypted_csv.decode()) examples = list(csv.reader(csv_data)) else: with open(log_file) as logs: examples = list(csv.reader(logs)) examples = examples[1:] #remove header for i, example in enumerate(examples): for j, (interface, cell) in enumerate( zip( app.interface.input_interfaces + app.interface.output_interfaces, example)): examples[i][j] = interface.restore_flagged(cell) return home_page(examples=examples, path=path)
def file(path): if app.interface.encrypt and isinstance(app.interface.examples, str) and path.startswith(app.interface.examples): with open(os.path.join(app.cwd, path), "rb") as encrypted_file: encrypted_data = encrypted_file.read() file_data = encryptor.decrypt( app.interface.encryption_key, encrypted_data) return send_file(io.BytesIO(file_data), attachment_filename=os.path.basename(path)) else: return send_file(os.path.join(app.cwd, path))
def encode_file_to_base64(f, encryption_key=None): with open(f, "rb") as file: encoded_string = base64.b64encode(file.read()) if encryption_key: encoded_string = encryptor.decrypt(encryption_key, encoded_string) base64_str = str(encoded_string, 'utf-8') mimetype = get_mimetype(f) return "data:" + (mimetype if mimetype is not None else "") + ";base64," + base64_str
def flag_data(input_data, output_data, flag_option=None): flag_path = os.path.join(app.cwd, app.interface.flagging_dir) csv_data = [] encryption_key = app.interface.encryption_key if app.interface.encrypt else None for i, interface in enumerate(app.interface.input_interfaces): csv_data.append( interface.save_flagged( flag_path, app.interface.config["input_interfaces"][i][1]["label"], input_data[i], encryption_key)) for i, interface in enumerate(app.interface.output_interfaces): csv_data.append( interface.save_flagged( flag_path, app.interface.config["output_interfaces"][i][1]["label"], output_data[i], encryption_key)) if flag_option: csv_data.append(flag_option) headers = [ interface[1]["label"] for interface in app.interface.config["input_interfaces"] ] headers += [ interface[1]["label"] for interface in app.interface.config["output_interfaces"] ] if app.interface.flagging_options is not None: headers.append("flag") log_fp = "{}/log.csv".format(flag_path) is_new = not os.path.exists(log_fp) if app.interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(app.interface.encryption_key, encrypted_csv) output.write(decrypted_csv.decode()) writer = csv.writer(output) if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(app.interface.encryption_key, output.getvalue().encode())) else: with open(log_fp, "a") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data)
def file(path): if (app.blocks.encrypt and isinstance(app.blocks.examples, str) and path.startswith(app.blocks.examples)): with open(safe_join(app.cwd, path), "rb") as encrypted_file: encrypted_data = encrypted_file.read() file_data = encryptor.decrypt(app.blocks.encryption_key, encrypted_data) return FileResponse(io.BytesIO(file_data), attachment_filename=os.path.basename(path)) else: if Path(app.cwd).resolve() in Path(path).resolve().parents: return FileResponse(Path(path).resolve())
def flag_data(input_data, output_data, flag_option=None, flag_index=None, username=None): flag_path = os.path.join(app.cwd, app.interface.flagging_dir) log_fp = "{}/log.csv".format(flag_path) encryption_key = app.interface.encryption_key if app.interface.encrypt else None is_new = not os.path.exists(log_fp) if flag_index is None: csv_data = [] for i, interface in enumerate(app.interface.input_components): csv_data.append( interface.save_flagged( flag_path, app.interface.config["input_components"][i]["label"], input_data[i], encryption_key)) for i, interface in enumerate(app.interface.output_components): csv_data.append( interface.save_flagged( flag_path, app.interface.config["output_components"][i] ["label"], output_data[i], encryption_key ) if output_data[i] is not None else "") if flag_option is not None: csv_data.append(flag_option) if username is not None: csv_data.append(username) if is_new: headers = [ interface["label"] for interface in app.interface.config["input_components"] ] headers += [ interface["label"] for interface in app.interface.config["output_components"] ] if app.interface.flagging_options is not None: headers.append("flag") if username is not None: headers.append("username") def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output) writer.writerows(content) return output.getvalue() if app.interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(app.interface.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output) if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(app.interface.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_fp, "a") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_fp) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_fp, "w") as csvfile: csvfile.write(file_content) with open(log_fp, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count
def flag( self, flag_data: List[Any], flag_option: Optional[str] = None, flag_index: Optional[int] = None, username: Optional[str] = None, ) -> int: flagging_dir = self.flagging_dir log_filepath = os.path.join(flagging_dir, "log.csv") is_new = not os.path.exists(log_filepath) if flag_index is None: csv_data = [] for component, sample in zip(self.components, flag_data): csv_data.append( component.save_flagged( flagging_dir, component.label, sample, self.encryption_key, ) if sample is not None else "") csv_data.append(flag_option if flag_option is not None else "") csv_data.append(username if username is not None else "") csv_data.append(str(datetime.datetime.now())) if is_new: headers = [component.label for component in self.components] + [ "flag", "username", "timestamp", ] def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") writer.writerows(content) return output.getvalue() if self.encryption_key: output = io.StringIO() if not is_new: with open(log_filepath, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(self.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_filepath, "wb") as csvfile: csvfile.write( encryptor.encrypt(self.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_filepath, "a", newline="") as csvfile: writer = csv.writer(csvfile, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_filepath) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_filepath, "w", newline="" ) as csvfile: # newline parameter needed for Windows csvfile.write(file_content) with open(log_filepath, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count
def test_same_pass(self): key = encryptor.get_key("test") data, _ = processing_utils.decode_base64_to_binary(BASE64_IMAGE) encrypted_data = encryptor.encrypt(key, data) decrypted_data = encryptor.decrypt(key, encrypted_data) self.assertEqual(data, decrypted_data)
def flag(self, interface, input_data, output_data, flag_option=None, flag_index=None, username=None): flagging_dir = self.flagging_dir log_fp = "{}/log.csv".format(flagging_dir) encryption_key = interface.encryption_key if interface.encrypt else None is_new = not os.path.exists(log_fp) output_only_mode = input_data is None if flag_index is None: csv_data = [] if not output_only_mode: for i, input in enumerate(interface.input_components): csv_data.append( input.save_flagged( flagging_dir, interface.config["input_components"][i]["label"], input_data[i], encryption_key)) for i, output in enumerate(interface.output_components): csv_data.append( output.save_flagged( flagging_dir, interface.config["output_components"][i] ["label"], output_data[i], encryption_key ) if output_data[i] is not None else "") if not output_only_mode: if flag_option is not None: csv_data.append(flag_option) if username is not None: csv_data.append(username) csv_data.append(str(datetime.datetime.now())) if is_new: headers = [] if not output_only_mode: headers += [ interface["label"] for interface in interface.config["input_components"] ] headers += [ interface["label"] for interface in interface.config["output_components"] ] if not output_only_mode: if interface.flagging_options is not None: headers.append("flag") if username is not None: headers.append("username") headers.append("timestamp") def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output) writer.writerows(content) return output.getvalue() if interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(interface.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output) if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(interface.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_fp, "a", newline="") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_fp) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_fp, "w", newline="" ) as csvfile: # newline parameter needed for Windows csvfile.write(file_content) with open(log_fp, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count