def connect_all(matrix, need_log=False):
"""
introduction: Integrate index and data from the two-dimensional matrix.
:param matrix: Data from input.
Type: Two-dimensional list(int).
:param need_log:
:return new_matrix: Data for output.
Type: Two-dimensional list(int).
"""
m = Monitor()
index_binary_length = int(len(str(bin(len(matrix)))) - 2)
if need_log:
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Add index in the binary matrix.")
new_matrix = []
for row in range(len(matrix)):
if need_log:
m.output(row + 1, len(matrix))
new_matrix.append(connect(row, matrix[row], index_binary_length))
m.restore()
del matrix, m
return new_matrix
def decode(self, dna_sequences, need_log=False):
"""
introduction: Decode DNA sequences to the data of binary file.
:param dna_sequences: The DNA sequence of len(matrix) rows.
Type: One-dimensional list(string).
:param need_log: Show the log.
:return matrix: The binary matrix corresponding to the DNA sequences.
Type: Two-dimensional list(int).
:return file_size: This refers to file size, to reduce redundant bits when transferring DNA to binary files.
Type: int
"""
if not dna_sequences:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"DNA sequence string set is not existing")
self.monitor.restore()
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Convert DNA sequences to binary matrix.")
matrix = self._convert_binaries(dna_sequences, need_log)
self.monitor.restore()
return matrix, self.file_size
def divide_all(matrix, need_log=False):
"""
introduction: Separate data from indexes in binary strings.
:param matrix: The DNA sequence of len(matrix) rows.
Type: Two-dimensional list(int).
:param need_log: need output log.
:returns index, datas: Obtained data sets and index sets in corresponding locations.
Type: One-dimensional list(int), Two-dimensional list(int).
"""
m = Monitor()
index_binary_length = int(len(str(bin(len(matrix)))) - 2)
if need_log:
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Divide index and data from binary matrix.")
indexs = []
datas = []
for row in range(len(matrix)):
if need_log:
m.output(row + 1, len(matrix))
index, data = divide(matrix[row], index_binary_length)
indexs.append(index)
datas.append(data)
m.restore()
del matrix, m
return indexs, datas
def write_dna_file(path, dna_sequences, need_log=False):
"""
introduction: Writing DNA sequence set to documents.
:param path: File path.
Type: string
:param dna_sequences: Generated DNA sequences.
Type: one-dimensional list(string)
:param need_log: choose to output log file or not.
"""
m = Monitor()
try:
with open(path, "w") as file:
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Write DNA sequences to file: " + path)
for row in range(len(dna_sequences)):
if need_log:
m.output(row + 1, len(dna_sequences))
file.write("".join(dna_sequences[row]) + "\n")
return dna_sequences
except IOError:
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"The file selection operation was not performed correctly. Please execute the operation again!"
)
def get_yyc_rules(need_log=False):
rules = []
temp_rule1 = ["".join(x) for x in itertools.product("01", repeat=4)]
temp_rule2 = ["".join(x) for x in itertools.product("01", repeat=16)]
m = Monitor()
if need_log:
# noinspection PyProtectedMember
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Find all the available Yin-Yang rules.")
count, step = 0, 0
for base in ["A", "T", "C", "G"]:
for rule1index in range(len(temp_rule1)):
for rule2index in range(len(temp_rule2)):
rule1 = list(map(int, list(temp_rule1[rule1index])))
rule2 = numpy.array(
list(map(int, list(temp_rule2[rule2index])))).reshape(
4, 4).tolist()
if _check(rule1, rule2):
rules.append(YYCRule(rule1, rule2, base, count))
count += 1
step += 1
if need_log:
m.output(step, len(temp_rule1) * len(temp_rule2) * 4)
return rules
def sort_order(indexes, data_set, need_log=False):
"""
introduction: Restore data in order of index.
:param indexes: The indexes of data set.
:param data_set: The disordered data set, the locations of this are corresponding to parameter "index".
:param need_log: need output log.
:returns matrix: Binary list in correct order.
Type: Two-dimensional list(int).
"""
m = Monitor()
if need_log:
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Restore data order according to index.")
# additional information checker
flag_index = 0
if max(indexes) > len(indexes):
while True:
if flag_index + 1 not in indexes:
# index to length
flag_index += 1
break
flag_index += 1
if need_log and flag_index > 0:
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"There are " + str(flag_index) + " required bit segments and " +
str(len(indexes) - flag_index) + " additional bit segments")
# noinspection PyUnusedLocal
if flag_index > 0:
matrix = [[0 for _ in range(len(data_set[0]))] for _ in range(flag_index)]
else:
matrix = [[0 for _ in range(len(data_set[0]))] for _ in range(len(indexes))]
for index in range(len(matrix)):
matrix[index] = data_set[indexes.index(index)]
if need_log:
m.output(index + 1, len(matrix))
m.restore()
del indexes, data_set, m
return matrix
def save_model(path, model):
"""
introduction: Save model to file.
:param path: The path of file.
Usually in the models directory.
:param model: Current model for encoding.
"""
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Save model to file: " + path)
with open(path, "wb") as file:
pickle.dump(model, file)
def write_all_from_binary(path, matrix, size, need_log=False):
"""
introduction: Writing binary matrix to document.
:param path: File path.
Type: string
:param matrix: A matrix in which each row represents a binary segment that will be used for DNA sequence generation.
Type: two-dimensional list(int)
:param size: This refers to file size, to reduce redundant bits when transferring DNA to binary files.
Type: int
:param need_log: choose to output log file or not.
"""
m = Monitor()
try:
with open(path, "wb+") as file:
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Write file from binary matrix: " + path)
# Change bit to byte (8 -> 1), and write a file as bytes
bit_index = 0
temp_byte = 0
for row in range(len(matrix)):
if need_log:
m.output(row + 1, len(matrix))
for col in range(len(matrix[0])):
bit_index += 1
temp_byte *= 2
temp_byte += matrix[row][col]
if bit_index == 8:
if size > 0:
file.write(struct.pack("B", int(temp_byte)))
bit_index = 0
temp_byte = 0
size -= 1
except IOError:
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"The file selection operation was not performed correctly. Please execute the operation again!"
)
def encode(self, matrix, size, need_log=False):
"""
introduction: Encode DNA sequences from the data of binary file.
:param matrix: Generated binary two-dimensional matrix.
The data of this matrix contains only 0 or 1 (non-char).
Type: int or bit.
:param size: This refers to file size, to reduce redundant bits when transferring DNA to binary files.
Type: int
:param need_log: Show the log.
:return dna_sequences: The DNA sequence of len(matrix) rows.
Type: list(list(char)).
"""
self.file_size = size
self.monitor.restore()
good_data_set, bad_data_set = self._divide_library(matrix, need_log)
self.monitor.restore()
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Random incorporation and validity testing.")
data_set = self._pairing(good_data_set, bad_data_set, need_log)
self.monitor.restore()
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Convert to DNA sequence string set.")
dna_sequences = self._synthesis_sequences(data_set, need_log)
self.monitor.restore()
return dna_sequences
def read_dna_file(path, need_log=False):
"""
introduction: Reading DNA sequence set from documents.
:param path: File path.
Type: string
:return dna_sequences: A corresponding DNA sequence string in which each row acts as a sequence.
Type: one-dimensional list(string)
:param need_log: need output log.
"""
m = Monitor()
dna_sequences = []
try:
with open(path, "r") as file:
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Read DNA sequences from file: " + path)
# Read current file by line
lines = file.readlines()
for index in range(len(lines)):
if need_log:
m.output(index + 1, len(lines))
line = lines[index]
dna_sequences.append(
[line[col] for col in range(len(line) - 1)])
return dna_sequences
except IOError:
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"The file selection operation was not performed correctly. Please execute the operation again!"
)
def get_yyc_rule_by_index(index, need_log=False):
"""
introduction: Get Yin and Yang rule of YYC by index (1536 types of rules)
:param index: rule index.
:param need_log: Show the log.
:return: YYC rule with [self.support_base, self.rule1, self.rule2].
"""
rules = get_yyc_rules(need_log)
if index < 0 or index >= len(rules):
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"We have " + str(len(rules)) + " rules, index " + str(index) +
" is wrong!")
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Current Rule is " + str(rules[index].get_info()) + ".")
return rules[index].get_info()
def encode(method,
input_path,
output_path,
model_path=None,
verify=None,
need_index=True,
segment_length=120,
need_log=False):
"""
introduction: Use the selected method, convert the binary file to DNA sequence
set and output the DNA sequence set.
:param method: Method under folder "methods/".
Type: Object.
:param input_path: The path of binary file you need to convert.
Type: String.
:param output_path: The path of DNA sequence set you need to use to .
Type: String.
:param model_path: The path of model file if you want to save
Type: String
:param verify: Error correction method under "methods/verifies/"
Type: Object.
:param need_index: Declare whether the binary sequence indexes are required
in the DNA sequences.
Type: bool.
:param segment_length: The cut length of DNA sequence.
Considering current DNA synthesis factors, we usually
set 120 bases as a sequence.
:param need_log: Show the log.
"""
if input_path is None or len(input_path) == 0:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"The input file path is invalid!")
if output_path is None or len(input_path) == 0:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"The output file path is invalid!")
input_matrix, size = data_handle.read_binary_from_all(
input_path, segment_length, need_log)
if need_index:
input_matrix = index_operator.connect_all(input_matrix, need_log)
if verify is not None:
input_matrix = verify.add_for_matrix(input_matrix, need_log)
dna_sequences = method.encode(input_matrix, size, need_log)
if model_path is not None:
model_saver.save_model(model_path, {
"method": method,
"verify": verify
})
data_handle.write_dna_file(output_path, dna_sequences, need_log)
def read_binary_from_all(path, segment_length=120, need_log=False):
"""
introduction: Reading binary matrix from document.
:param path: File path.
Type: string
:param segment_length: The binary segment length used for DNA sequence generation.
Considering current DNA synthesis technique limitation,
we usually set 120 as default segment length.
:param need_log: choose to output log file or not.
:return matrix: A matrix in which each row represents a binary segment that will be used for DNA sequence generation.
Type: two-dimensional list(int)
"""
m = Monitor()
try:
# Open selected file
with open(path, mode="rb") as file:
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Read binary matrix from file: " + path)
size = os.path.getsize(path)
# Set init storage matrix
matrix = [[0 for _ in range(segment_length)]
for _ in range(math.ceil(size * 8 / segment_length))]
row = 0
col = 0
for byte_index in range(size):
if need_log:
m.output(byte_index + 1, size)
# Read a file as bytes
one_byte = file.read(1)
element = list(
map(
int,
list(
str(bin(struct.unpack(
"B", one_byte)[0]))[2:].zfill(8))))
for bit_index in range(8):
matrix[row][col] = element[bit_index]
col += 1
if col == segment_length:
col = 0
row += 1
if int(len(str(bin(len(matrix)))) - 2) * 7 > segment_length:
if need_log:
log.output(
log.WARN, str(__name__),
str(sys._getframe().f_code.co_name),
"The proportion of index in whole sequence may be high. \n"
"It is recommended to increase the length of output DNA sequences "
"or to divide the file into more segment pools")
return matrix, size
except IOError:
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"The file selection operation was not performed correctly. Please execute the operation again!"
)
def _init_check(self):
"""
introduction: The verification of initialization parameters.
"""
# Check support bases
for index in range(len(self.support_bases)):
if (self.support_bases[index] != "A"
and self.support_bases[index] != "T"
and self.support_bases[index] != "C"
and self.support_bases[index] != "G"):
log.output(
log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Only A, T, C, and G can be included as support bases, "
"but the support bases[" + str(index) +
"] has been detected as " +
str(self.support_bases[index] + "!"))
if len(self.support_bases) < self.support_spacing + 1:
log.output(
log.ERROR, str(__name__), str(sys._getframe().f_code.co_name),
"The count of support base needs to be more than support spacing!"
)
# Check base reference (rule 1)
for index in range(len(self.base_reference)):
if self.base_reference[index] != 0 and self.base_reference[
index] != 1:
log.output(
log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Only 0 and 1 can be included for base reference, and base_reference["
+ str(index) + "] has been detected as " +
str(self.base_reference[index]) + "!")
if sum(self.base_reference) != 2:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Wrong correspondence between base and binary data!")
positions = []
for i in range(len(self.base_reference)):
if self.base_reference[i] == 1:
positions.append(i)
for i in range(len(self.base_reference)):
if self.base_reference[i] != 1:
positions.append(i)
# Check current code matrix (rule 2)
for row in range(len(self.current_code_matrix)):
for col in range(len(self.current_code_matrix[row])):
if self.current_code_matrix[row][
col] != 0 and self.current_code_matrix[row][col] != 1:
log.output(
log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Only 0 and 1 can be included in the current code matrix, and the current code matrix ["
+ str(row) + ", " + str(col) +
"] has been detected as " +
str(self.current_code_matrix[row][col]) + "!")
for row in range(len(self.current_code_matrix)):
left = self.current_code_matrix[row][positions[0]]
right = self.current_code_matrix[row][positions[1]]
if left + right == 1 and left * right == 0:
continue
else:
log.output(
log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Wrong current code matrix, the error locations are [" +
str(row) + ", " + str(positions[0]) + "] and [" +
str(row) + ", " + str(positions[1]) + "]! " +
"It is required by rule that these two values will have sum of 1 and product of 0."
)
left = self.current_code_matrix[row][positions[2]]
right = self.current_code_matrix[row][positions[3]]
if left + right == 1 and left * right == 0:
continue
else:
log.output(
log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Wrong current code matrix, the error locations are [" +
str(row) + ", " + str(positions[2]) + "] and [" +
str(row) + ", " + str(positions[3]) + "]! " +
"It is required by rule that these two values will have sum of 1 and product of 0."
)
# Check max ratio
if self.max_ratio <= 0.5 or self.max_ratio >= 1:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Wrong max ratio (" + str(self.max_ratio) + ")!")
def _divide_library(self, matrix, need_log):
"""
introduction: Separate 'good' and 'bad' data from total data, and splice index and data as a list.
:param matrix: Generated binary two-dimensional matrix
The data of this matrix contains only 0 or 1 (non-char).
Type: int or bit
:param need_log: Show the log.
:returns good_data_set, bad datas: 'good' and 'bad' data from total data
Type: list(int)
"""
if need_log:
log.output(log.NORMAL, str(__name__),
str(sys._getframe().f_code.co_name),
"Separate 'good' data from 'bad' data.")
bad_indexes = []
for row in range(len(matrix)):
if numpy.sum(matrix[row]) > len(matrix[row]) * self.max_ratio \
or numpy.sum(matrix[row]) < len(matrix[row]) * (1 - self.max_ratio):
bad_indexes.append(row)
if len(matrix) < len(bad_indexes) * 5:
if need_log:
log.output(
log.WARN, str(__name__),
str(sys._getframe().f_code.co_name),
"There may be a large number of sequences that are difficult for synthesis or sequencing. "
+
"We recommend you to re-select the rule or take a new run."
)
if len(bad_indexes) == 0 and len(matrix) == 0:
return [], []
elif len(bad_indexes) == 0:
good_data_set = []
for row in range(len(matrix)):
if need_log:
self.monitor.output(row + 1, len(matrix))
good_data_set.append(matrix[row])
return good_data_set, []
elif len(bad_indexes) == len(matrix):
bad_data_set = []
for row in range(len(matrix)):
if need_log:
self.monitor.output(row + 1, len(matrix))
bad_data_set.append(matrix[row])
return [], bad_data_set
else:
good_data_set = []
bad_data_set = []
for row in range(len(matrix)):
if need_log:
self.monitor.output(row + 1, len(matrix))
if row in bad_indexes:
bad_data_set.append(matrix[row])
else:
good_data_set.append(matrix[row])
return good_data_set, bad_data_set
def _pairing(self, good_data_set, bad_data_set, need_log):
"""
introduction: Match 'good' data with 'bad' data, to ensure that the overall data is better.
If there are only 'good' or 'bad' data left, they will be selected to pair with each other.
:param good_data_set: Generated binary two-dimensional matrix, the repetition rate of 0 or 1 is related low.
Type: Two-dimensional list(int)
:param bad_data_set: Generated binary two-dimensional matrix, the repetition rate of 0 or 1 is related high.
Type: Two-dimensional list(int)
:param need_log: Show the log.
:returns data_set: Matched results
Type: Two-dimensional list(int)
"""
data_set = []
if good_data_set is None and bad_data_set is None:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"YYC did not receive matrix data!")
total_count = len(good_data_set) + len(bad_data_set)
index_bit_length = int(len(str(bin(total_count))) - 2)
search_counts = [0 for _ in range(self.search_count + 1)]
additional = 0
while len(good_data_set) + len(bad_data_set) > 0:
if len(good_data_set) > 0 and len(bad_data_set) > 0:
fixed_list = random.sample(bad_data_set, 1)[0]
bad_data_set.remove(fixed_list)
another_list, is_upper, search_count = self._searching_results(
fixed_list, good_data_set, index_bit_length, total_count)
if search_count >= 0:
good_data_set.remove(another_list)
search_counts[search_count] += 1
else:
additional += 1
if is_upper:
data_set.append(fixed_list)
data_set.append(another_list)
else:
data_set.append(another_list)
data_set.append(fixed_list)
elif len(good_data_set) > 0:
fixed_list = random.sample(good_data_set, 1)[0]
good_data_set.remove(fixed_list)
another_list, is_upper, search_count = self._searching_results(
fixed_list, good_data_set, index_bit_length, total_count)
if search_count >= 0:
good_data_set.remove(another_list)
search_counts[search_count] += 1
else:
additional += 1
if is_upper:
data_set.append(fixed_list)
data_set.append(another_list)
else:
data_set.append(another_list)
data_set.append(fixed_list)
elif len(bad_data_set) > 0:
fixed_list = random.sample(bad_data_set, 1)[0]
bad_data_set.remove(fixed_list)
another_list, is_upper, search_count = self._searching_results(
fixed_list, bad_data_set, index_bit_length, total_count)
if search_count >= 0:
bad_data_set.remove(another_list)
search_counts[search_count] += 1
else:
additional += 1
if is_upper:
data_set.append(fixed_list)
data_set.append(another_list)
else:
data_set.append(another_list)
data_set.append(fixed_list)
else:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"Wrong pairing for Yin-Yang Code!")
if need_log:
self.monitor.output(
total_count - (len(good_data_set) + len(bad_data_set)),
total_count)
results = {}
for index, count in enumerate(search_counts):
results[index] = count
if need_log:
log.output(
log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Number of additional bit segment is " + str(additional) +
" in original " + str(total_count) + " bit segments.")
log.output(
log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"In addition, the actual search counts is " + str(results))
del good_data_set, bad_data_set
return data_set
def decode(method=None,
model_path=None,
input_path=None,
output_path=None,
verify=None,
has_index=True,
need_log=False):
"""
introduction: Use the selected method, convert DNA sequence set to the binary
file and output the binary file.
:param method: Method under folder "methods/".
If you have model file, you can use this function with out
method.
Type: Object.
:param input_path: The path of DNA sequence set you need to convert.
Type: String.
:param output_path: The path of binary file consistent with previous
documents.
Type: String.
:param model_path: The path of model file if you want to save
Type: String
:param verify: Error correction method under "methods/verifies/"
Type: Object.
:param has_index: Declare whether the DNA sequences contain binary sequence
indexes.
Type: bool.
:param need_log: Show the log.
"""
if method is None and model_path is None:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"The method you select does not exist!")
else:
if input_path is None or len(input_path) == 0:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"The input file path is not valid!")
if output_path is None or len(output_path) == 0:
log.output(log.ERROR, str(__name__),
str(sys._getframe().f_code.co_name),
"The output file path is not valid!")
if model_path is not None:
model = model_saver.load_model(model_path)
method = model.get("method")
verify = model.get("verify")
dna_sequences = data_handle.read_dna_file(input_path, need_log)
output_matrix, size = method.decode(dna_sequences, need_log)
if verify is not None:
output_matrix = verify.verify_for_matrix(output_matrix, need_log)
if has_index:
indexes, data_set = index_operator.divide_all(
output_matrix, need_log)
output_matrix = index_operator.sort_order(indexes, data_set,
need_log)
data_handle.write_all_from_binary(output_path, output_matrix, size,
need_log)
def load_model(path):
log.output(log.NORMAL, str(__name__), str(sys._getframe().f_code.co_name),
"Load model from file: " + path)
with open(path, "rb") as file:
return pickle.load(file)