def flag_data(input_data, output_data, flag_option=None): flag_path = os.path.join(app.cwd, app.interface.flagging_dir) csv_data = [] encryption_key = app.interface.encryption_key if app.interface.encrypt else None for i, interface in enumerate(app.interface.input_interfaces): csv_data.append( interface.save_flagged( flag_path, app.interface.config["input_interfaces"][i][1]["label"], input_data[i], encryption_key)) for i, interface in enumerate(app.interface.output_interfaces): csv_data.append( interface.save_flagged( flag_path, app.interface.config["output_interfaces"][i][1]["label"], output_data[i], encryption_key)) if flag_option: csv_data.append(flag_option) headers = [ interface[1]["label"] for interface in app.interface.config["input_interfaces"] ] headers += [ interface[1]["label"] for interface in app.interface.config["output_interfaces"] ] if app.interface.flagging_options is not None: headers.append("flag") log_fp = "{}/log.csv".format(flag_path) is_new = not os.path.exists(log_fp) if app.interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(app.interface.encryption_key, encrypted_csv) output.write(decrypted_csv.decode()) writer = csv.writer(output) if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(app.interface.encryption_key, output.getvalue().encode())) else: with open(log_fp, "a") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data)
def decode_base64_to_file(encoding, encryption_key=None): data, extension = decode_base64_to_binary(encoding) if extension is None: file_obj = tempfile.NamedTemporaryFile(delete=False) else: file_obj = tempfile.NamedTemporaryFile(delete=False, suffix="."+extension) if encryption_key is not None: data = encryptor.encrypt(encryption_key, data) file_obj.write(data) file_obj.flush() return file_obj
def decode_base64_to_file(encoding, encryption_key=None, filename_prefix=""): data, extension = decode_base64_to_binary(encoding) if extension is None: file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=filename_prefix) else: file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=filename_prefix, suffix="."+extension) if encryption_key is not None: data = encryptor.encrypt(encryption_key, data) #print("saving to ", file_obj.name) file_obj.write(data) file_obj.flush() return file_obj
def decode_base64_to_file(encoding, encryption_key=None, file_path=None): data, extension = decode_base64_to_binary(encoding) prefix = None if file_path is not None: filename = os.path.basename(file_path) prefix = filename if "." in filename: prefix = filename[0 : filename.index(".")] extension = filename[filename.index(".") + 1 :] if extension is None: file_obj = tempfile.NamedTemporaryFile(delete=False, prefix=prefix) else: file_obj = tempfile.NamedTemporaryFile( delete=False, prefix=prefix, suffix="." + extension ) if encryption_key is not None: data = encryptor.encrypt(encryption_key, data) file_obj.write(data) file_obj.flush() return file_obj
def flag_data(input_data, output_data, flag_option=None, flag_index=None, username=None): flag_path = os.path.join(app.cwd, app.interface.flagging_dir) log_fp = "{}/log.csv".format(flag_path) encryption_key = app.interface.encryption_key if app.interface.encrypt else None is_new = not os.path.exists(log_fp) if flag_index is None: csv_data = [] for i, interface in enumerate(app.interface.input_components): csv_data.append( interface.save_flagged( flag_path, app.interface.config["input_components"][i]["label"], input_data[i], encryption_key)) for i, interface in enumerate(app.interface.output_components): csv_data.append( interface.save_flagged( flag_path, app.interface.config["output_components"][i] ["label"], output_data[i], encryption_key ) if output_data[i] is not None else "") if flag_option is not None: csv_data.append(flag_option) if username is not None: csv_data.append(username) if is_new: headers = [ interface["label"] for interface in app.interface.config["input_components"] ] headers += [ interface["label"] for interface in app.interface.config["output_components"] ] if app.interface.flagging_options is not None: headers.append("flag") if username is not None: headers.append("username") def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output) writer.writerows(content) return output.getvalue() if app.interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(app.interface.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output) if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(app.interface.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_fp, "a") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_fp) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_fp, "w") as csvfile: csvfile.write(file_content) with open(log_fp, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count
def flag( self, flag_data: List[Any], flag_option: Optional[str] = None, flag_index: Optional[int] = None, username: Optional[str] = None, ) -> int: flagging_dir = self.flagging_dir log_filepath = os.path.join(flagging_dir, "log.csv") is_new = not os.path.exists(log_filepath) if flag_index is None: csv_data = [] for component, sample in zip(self.components, flag_data): csv_data.append( component.save_flagged( flagging_dir, component.label, sample, self.encryption_key, ) if sample is not None else "") csv_data.append(flag_option if flag_option is not None else "") csv_data.append(username if username is not None else "") csv_data.append(str(datetime.datetime.now())) if is_new: headers = [component.label for component in self.components] + [ "flag", "username", "timestamp", ] def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") writer.writerows(content) return output.getvalue() if self.encryption_key: output = io.StringIO() if not is_new: with open(log_filepath, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(self.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_filepath, "wb") as csvfile: csvfile.write( encryptor.encrypt(self.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_filepath, "a", newline="") as csvfile: writer = csv.writer(csvfile, quoting=csv.QUOTE_NONNUMERIC, quotechar="'") if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_filepath) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_filepath, "w", newline="" ) as csvfile: # newline parameter needed for Windows csvfile.write(file_content) with open(log_filepath, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count
def test_same_pass(self): key = encryptor.get_key("test") data, _ = processing_utils.decode_base64_to_binary(BASE64_IMAGE) encrypted_data = encryptor.encrypt(key, data) decrypted_data = encryptor.decrypt(key, encrypted_data) self.assertEqual(data, decrypted_data)
def flag(self, interface, input_data, output_data, flag_option=None, flag_index=None, username=None): flagging_dir = self.flagging_dir log_fp = "{}/log.csv".format(flagging_dir) encryption_key = interface.encryption_key if interface.encrypt else None is_new = not os.path.exists(log_fp) output_only_mode = input_data is None if flag_index is None: csv_data = [] if not output_only_mode: for i, input in enumerate(interface.input_components): csv_data.append( input.save_flagged( flagging_dir, interface.config["input_components"][i]["label"], input_data[i], encryption_key)) for i, output in enumerate(interface.output_components): csv_data.append( output.save_flagged( flagging_dir, interface.config["output_components"][i] ["label"], output_data[i], encryption_key ) if output_data[i] is not None else "") if not output_only_mode: if flag_option is not None: csv_data.append(flag_option) if username is not None: csv_data.append(username) csv_data.append(str(datetime.datetime.now())) if is_new: headers = [] if not output_only_mode: headers += [ interface["label"] for interface in interface.config["input_components"] ] headers += [ interface["label"] for interface in interface.config["output_components"] ] if not output_only_mode: if interface.flagging_options is not None: headers.append("flag") if username is not None: headers.append("username") headers.append("timestamp") def replace_flag_at_index(file_content): file_content = io.StringIO(file_content) content = list(csv.reader(file_content)) header = content[0] flag_col_index = header.index("flag") content[flag_index][flag_col_index] = flag_option output = io.StringIO() writer = csv.writer(output) writer.writerows(content) return output.getvalue() if interface.encrypt: output = io.StringIO() if not is_new: with open(log_fp, "rb") as csvfile: encrypted_csv = csvfile.read() decrypted_csv = encryptor.decrypt(interface.encryption_key, encrypted_csv) file_content = decrypted_csv.decode() if flag_index is not None: file_content = replace_flag_at_index(file_content) output.write(file_content) writer = csv.writer(output) if flag_index is None: if is_new: writer.writerow(headers) writer.writerow(csv_data) with open(log_fp, "wb") as csvfile: csvfile.write( encryptor.encrypt(interface.encryption_key, output.getvalue().encode())) else: if flag_index is None: with open(log_fp, "a", newline="") as csvfile: writer = csv.writer(csvfile) if is_new: writer.writerow(headers) writer.writerow(csv_data) else: with open(log_fp) as csvfile: file_content = csvfile.read() file_content = replace_flag_at_index(file_content) with open(log_fp, "w", newline="" ) as csvfile: # newline parameter needed for Windows csvfile.write(file_content) with open(log_fp, "r") as csvfile: line_count = len([None for row in csv.reader(csvfile)]) - 1 return line_count