def close(self):
if self._buffer:
self._buffer.close()
self._wrapped_stream = None
IOBase.close(self)
def trim(barcode_distance: int,
barcode_sequence: str,
reverse: bool = False,
inStream: io.IOBase = stdin,
outStream: io.IOBase = stdout,
mateStream: io.IOBase = None,
verbose: bool = True,
logStream: io.IOBase = stderr) -> (int, int):
"""
Trims barcodes from reads in a fastq file
:param inStream: A file or stream handle to read input data
:param mateStream: A file or stream handle to read mate input data
:param outStream: A file or stream handle to output data
:param barcode_distance: The maximum number of differences from the barcode sequence before the read is rejected. Set to negative to output rejected reads only.
:param barcode_sequence: The IUPAC barcode sequence to match against.
:param reverse: Set to true to read sequences in reverse, looking for the barcode at the other end.
:param verbose: Provide verbose output while processing
:return:
"""
printPrefix = "PRODUSE-TRIM"
if verbose:
logStream.write("\t".join(
[printPrefix, time.strftime('%X'), "Starting...\n"]))
record1 = FastqRecord()
mated = mateStream is not None
if mated:
record2 = FastqRecord()
invert = False
count = 0 # Record counter
discard = 0 # Discarded records counter
if barcode_distance < 0:
invert = True
barcode_distance *= -1
while record1.read(inStream) and (not mated or record2.read(mateStream)):
"""
if verbose:
logStream.write("\x1b[F\x1b[2K\r{file}\tWorking on record: {record}\tRecords processed: {total}\n".format(file=outStream.name if hasattr(outStream, 'name') else 'Streaming', record=record1.name, total=count))
"""
count += 1
# Store barcode in sequence description
barcode1 = (record1.seq[-len(barcode_sequence):]
if reverse else record1.seq[:len(barcode_sequence)])
if mated:
barcode2 = (record2.seq[-len(barcode_sequence):]
if reverse else record2.seq[:len(barcode_sequence)])
else:
barcode2 = ''
record1.desc1 += '\tBC:Z:' + barcode1 + barcode2
if mated:
record2.desc1 += '\tBC:Z:' + barcode2 + barcode1
for record, barcode in ((record1, barcode1),
(record2,
barcode2)) if mated else ((record1,
barcode1)):
mismatch = 0
mismatch += sum(
not IUPACMatch(bc, sq)
for bc, sq in zip(barcode_sequence,
reversed(barcode) if reverse else barcode))
if len(barcode) < len(barcode_sequence):
mismatch += len(barcode_sequence) - len(barcode)
if invert != (mismatch > barcode_distance): # XOR
discard += 1
# Keep empty record to make BWA happy about having read mate
record.seq = ''
record.qual = ''
else:
record.trim(0 if reverse else len(barcode_sequence),
len(barcode_sequence) if reverse else 0)
record.write(outStream)
# Provide status updates
if verbose and count % 1000 == 0:
if mated:
logStream.write("\t".join([
printPrefix,
time.strftime('%X'),
"Read Pairs Trimmed: %s Discard Rate: %s \r" %
(count, discard / count)
]))
else:
logStream.write("\t".join([
printPrefix,
time.strftime('%X'),
"Reads Trimmed: %s Discard Rate: %s \r" %
(count, discard / count)
]))
if verbose:
logStream.write("\t".join(
["\n" + printPrefix,
time.strftime('%X'), "Trimming Complete\n"]))
outStream.close()
return discard, count
def get_file_data_and_close_file(file_instance: io.IOBase) -> bytes:
file_data = file_instance.read()
file_instance.close()
return file_data
def close(self):
if self._buffer:
self._buffer.close()
self._wrapped_stream = None
IOBase.close(self)
def extract_steganography(input_file: io.IOBase, output_file: io.IOBase,
auth_key: str) -> int:
try:
image = Image.open(input_file)
except Image.UnidentifiedImageError:
logger.exception(f"Not an image file!")
return 1
header = extract_header(image)
pix = image.load()
y_dim = image.size[1]
data_length = Header.header_length + header.data_length
x, y, count = 0, 0, 0
result_data = b""
bit_loc = header.density - 1
# Attempt to read input_file
while len(result_data) < data_length:
byte = 0
# Read every single bit
# Iterate through every single bit of the byte
for i in range(8):
# If bit is set, set the corresponding bit of 'byte'
if BitOps.is_bit_set(pix[x, y][count], bit_loc):
byte = BitOps.set_bit(byte, i)
# Move to the next bit by decrement bit index
bit_loc -= 1
# If all readable bits of the colour integer are consumed
if bit_loc == -1:
# Move to the next RGB and reset the bit index
count += 1
bit_loc = header.density - 1
# If the entire pixel is read
if count == 3:
# Move to the next pixel in the row and reset the count
count = 0
y += 1
# If the entire row of pixels is read
if y == y_dim:
# Move to the next row and reset row index
y = 0
x += 1
# Convert the single byte (integer) to bytes
# By design, the resulting input_file is strictly stored in 1 byte
# and endianness does not matter since it is only 1 byte
result_data += byte.to_bytes(1, "big")
# Strip header by slicing its known length
result_data = result_data[Header.header_length:]
result_data = fix(result_data, False)
# Decrypt input_file
crypto = Cryptographer.make_decrypter(header.salt, auth_key)
try:
# 5. Store decrypted input_file
result_data = crypto.decrypt(result_data)
except cryptography.fernet.InvalidToken:
logger.exception("Invalid authentication key!")
return 1
# If compressed (as indicated by the header), decompress it
if header.compression > 0:
result_data = bz2.decompress(result_data)
# Write input_file to output_file file objects
# Iterate through all file objects
try:
output_file.write(result_data)
output_file.close()
except IOError:
logger.exception("Data cannot be writen")
return 1
return 0
def write_steganography(input_file: io.IOBase,
image_file: Image.Image,
output_file: io.IOBase,
auth_key: str,
compression: int,
density: int,
show_image_on_completion: bool,
config: dict[str, ...] = None) -> int:
auth_key = config["default_auth_key"] if auth_key is None else auth_key
compression = config["default_compression"] if compression not in config[
"available_compression"] else compression
density = config["default_density"] if density not in config[
"available_density"] else density
show_image_on_completion = config["flag_show_image_on_completion"] \
if show_image_on_completion is None else show_image_on_completion
input_file.seek(0)
data = input_file.read()
if data is None:
logger.error("Input file is not readable!")
return 1
if len(data) == 0:
logger.error("Input file is empty or exhausted!")
return 1
if compression > 0:
# Compress using the builtin bzip2 library
data = bz2.compress(data, compresslevel=compression)
crypto = Cryptographer.make_encrypter(
Cryptographer.encode_salt(Cryptographer.make_salt()), auth_key)
data = crypto.encrypt(data)
data = fix(data)
# Craft the finished input_file
header = build_header(
data_length=len(data),
compression=compression,
density=density,
salt=crypto.get_salt_string(),
)
# 2. Serialise header and prepend input_file with header
data = bytes(header.header, "utf-8") + data
try:
pix = image_file.load()
except Exception or BaseException:
logger.exception("Cannot load image_file file!")
return 1
x_dim, y_dim = image_file.size
# Make sure there are enough space to store all bits
no_of_pixel = x_dim * y_dim
no_of_rgb = no_of_pixel * 3
no_of_storable_bit = no_of_rgb * density
no_of_stored_bit = len(data) * 8
if no_of_storable_bit < no_of_stored_bit:
# If there are not enough, raise error
logger.error("Data is too big to be stored!")
return 1
x, y, count, bit_loc = 0, 0, 0, density - 1
current_pix = list(pix[0, 0])
# Firstly, iterate through all the bytes to be written
for byte in data:
# Secondly, iterate through all the bits of the given byte
for i in range(8):
# Thirdly, check if the bit is set
# If bit is set
if BitOps.is_bit_set(byte, i):
# Check if the bit at the current location in the image_file is set
# If unset then set it, otherwise unchanged
current_pix[count] = BitOps.set_bit(current_pix[count],
bit_loc)
# If bit is unset
else:
# Check if the bit at the current location in the image_file is set
# If set then unset it, otherwise unchanged
current_pix[count] = BitOps.unset_bit(current_pix[count],
bit_loc)
# Move to the next bit
# by decrementing index
bit_loc -= 1
# If reached the final bit
if bit_loc == -1:
# Move to the next integer
# by incrementing the count
count += 1
# Reset density
bit_loc = density - 1
# If reached the last RGB
if count == 3:
# Save pixel
pix[x, y] = tuple(current_pix)
# Reset count
count = 0
y += 1
# If the entire row of pixel is written
if y == y_dim:
# Move on to the next row and reset
y = 0
x += 1
# Request new pixel to be written
current_pix = list(pix[x, y])
try:
image_file.save(output_file, "png")
except OSError:
logger.exception("Cannot save image_file to output_file file!")
return 1
if show_image_on_completion:
image_file.show("Demo")
input_file.close()
image_file.close()
output_file.close()
return 0
def crashing_function(video_handle: io.IOBase, random_arg: int) -> {
"results": str
}:
video_handle.close()
raise ValueError("Some error")
return {"results": "okay"}
def large_file_function_wait(video_handle: io.IOBase, random_arg: int) -> {
"results": io.IOBase
}:
video_handle.close()
time.sleep(10)
return {"results": open(video_handle.name, 'rb')}