def __init__(self, window: Window):
super().__init__(window)
from constants import BUTTON_SIZE
window_size = window.get_size()
window_center = (window_size[0] / 2, window_size[1] / 2)
dpi_factor = window.hidpi_factor
button_size = (BUTTON_SIZE[0] * dpi_factor,
BUTTON_SIZE[1] * dpi_factor)
button_font = Font('sans-serif', 16, dpi_factor)
self.bg_image = util.load_image('assets/background.png')
back_btn = Button(window,
'[ Back ]',
Vector(window_center[0] - button_size[0] / 2,
window_size[1] * 2 / 3),
Vector(*button_size),
Color(0, 102, 255),
Color(255, 255, 255),
Color(0, 80, 230),
Color(255, 255, 255),
font=button_font)
back_btn.set_click_handler(self.back)
self.children.append(back_btn)
def __init__(self,
window: Window,
text: str,
pos: Vector,
size: Vector = Vector(150, 50),
bg: Color = Color(200, 200, 200),
fg: Color = Color(20, 20, 20),
bg_over: Color = Color(220, 220, 220),
fg_over: Color = Color(20, 20, 20),
border_size: int = 0,
font: Font = Font('sans-serif', 15, HIDPI_FACTOR)):
super().__init__(window)
self._click_handler = None
self.text = text
self.pos = pos
self.size = size
# Initial colours
self.bg = bg
self.fg = fg
self.bg_over = bg_over
self.fg_over = fg_over
# Extra
self.font = font
self.border_size = border_size
# Center
self.center = (self.pos[0] + self.size[0] / 2,
self.pos[1] + self.size[1] / 2)
# Get corners
self.bounds = BoundingBox(pos, pos + size)
def __init__(self, window: Window):
super().__init__(window)
from constants import BUTTON_SIZE
self.window_size = window.get_size()
self.window_center = (self.window_size[0] / 2, self.window_size[1] / 2)
dpi_factor = window.hidpi_factor
button_size = (BUTTON_SIZE[0] * dpi_factor,
BUTTON_SIZE[1] * dpi_factor)
button_font = Font('sans-serif', 16, dpi_factor)
self.bg_image = util.load_image('assets/background.png')
self.bg_size = (self.bg_image.get_width(), self.bg_image.get_height())
self.bg_center = (self.bg_size[0] / 2, self.bg_size[1] / 2)
self.logo = util.load_image('assets/logo.png')
self.logo_size = (self.logo.get_width(), self.logo.get_height())
self.logo_center = (self.logo_size[0] / 2, self.logo_size[1] / 2)
self.max_score = 0
self.last_active_level = None
self.text_font = Font('monospace', 20, window.hidpi_factor)
self.text_font_color = Color(255, 255, 255)
# Template to create new button
start_btn = Button(window,
'[ Start ]',
Vector(self.window_center[0] - button_size[0] / 2,
self.window_center[1] - button_size[1]),
Vector(*button_size),
Color(0, 102, 255),
Color(255, 255, 255),
Color(0, 80, 230),
Color(255, 255, 255),
font=button_font)
start_btn.set_click_handler(self.start)
self.children.append(start_btn)
help_btn = Button(window,
'[ Help ]',
Vector(self.window_center[0] - button_size[0] / 2,
self.window_center[1] + button_size[1]),
Vector(*button_size),
Color(0, 102, 255),
Color(255, 255, 255),
Color(0, 80, 230),
Color(255, 255, 255),
font=button_font)
help_btn.set_click_handler(self.help)
self.children.append(help_btn)
def __init__(self, gb, seq_keys, gb_dir, num_cores, minlength, maxlength, length_thres=0.5, threshold=0.75, evalue=(1.0/10**10)):
"""
Input: gb dictionary of SeqRecords, keys to all sequences, and an optional threshold for clustering.
Output: a list of cluster files from UCLUST
"""
ClusterBuilder.__init__(self, seq_keys)
self.seq_keys = seq_keys
self.threshold = threshold
color = Color()
if not os.path.exists("uclusters"):
os.makedirs("uclusters")
# write sequences to fasta
sequences = []
for seq_key in seq_keys:
record = gb[seq_key]
record.description = record.annotations["organism"] + " " + record.description
if "sp." not in record.annotations["organism"]:
sequences.append(record)
file_name = "_sumac"
f = open(file_name, "wb")
SeqIO.write(sequences, f, 'fasta')
f.close()
with open("_sumac", "r") as f, open("_sumac_filtered", "w") as fout:
for l in f:
if ">" in l:
l = l.replace(" ", "_")
fout.write(l)
# call UCLUST
sort_sequences = ["usearch", "-sortbylength", "_sumac_filtered", "-fastaout", "_sumac_sorted",
"-minseqlength", str(minlength), "-maxseqlength", str(maxlength)]
uclust = ["usearch", "-cluster_fast", "_sumac_sorted", "-id", str(threshold),
"-minsl", str(length_thres), "-strand", "both", "-threads", str(num_cores),
"-clusters", "uclusters/", "-fulldp", "-evalue", str(evalue)]
try:
subprocess.check_call(sort_sequences)
subprocess.check_call(uclust)
except CalledProcessError as e:
print(color.red + "UCLUST error: " + str(e) + color.done)
print(color.red + "Trying SLINK instead..." + color.done)
self.error = True
return
except OSError as e:
print(color.red + "UCLUST is not installed correctly." + color.done)
print(color.red + "OS error: " + str(e) + color.done)
print(color.red + "Trying SLINK instead..." + color.done)
self.error = True
return
finally:
if os.path.exists("_sumac"):
subprocess.check_call(["rm", "_sumac"])
if os.path.exists("_sumac_filtered"):
subprocess.check_call(["rm", "_sumac_filtered"])
if os.path.exists("_sumac_sorted"):
subprocess.check_call(["rm", "_sumac_sorted"])
cluster_files = [ f for f in listdir("uclusters/") if isfile(join("uclusters/", f)) ]
for f in cluster_files:
self.clusters.append(f)
def calculate_PD_parallel(self, num_cores):
"""
Method to calculate the fraction of triples, a measure of partial decisiveness (PD).
See: Sanderson, M.J., McMahon, M.M. & Steel, M., 2010. BMC evolutionary biology, 10.
"""
color = Color()
lock = multiprocessing.Lock()
manager = multiprocessing.Manager()
#already_compared = manager.list()
#dist_matrix = manager.list()
otus_shared = manager.dict()
otus_shared = otus
decisive_triples = manager.Value('i', 0)
total_triples = manager.Value('i', 0)
total = self.binomial_coefficient(len(self.otus), 3)
for i in range(num_cores):
p = multiprocessing.Process(target=calculate_PD_worker,
args=(lock, i, num_cores,
decisive_triples, total_triples,
total, otus_shared))
p.start()
processes.append(p)
for p in processes:
p.join()
self.otus = otus_shared
sys.stdout.write("\r" + color.blue + "Calculating PD: " + color.red +
"100.00% " + color.blue + "finished\n" + color.done)
sys.stdout.flush()
return round(decisive_triples / float(total_triples), 2)
def __init__(self, gb, seq_keys, length_threshold, gb_dir, num_cores):
"""
Takes as input a dictionary of SeqRecords gb and the keys to all sequences.
length_threshold is the threshold of sequence length percent similarity to cluster taxa.
For example if length_threshold = 0.25, and one sequence has
length 100, the other sequence must have length 75 to 125. If the lengths are not similar
enough the distance is set to 50 (which keeps them from being clustered).
Generates a 2 dimensional list of distances. Distances are blastn e-values.
"""
lock = multiprocessing.Lock()
manager = multiprocessing.Manager()
already_compared = manager.list()
dist_matrix = manager.list()
row = []
for i in range(len(seq_keys)):
row.append(99)
for i in range(len(seq_keys)):
dist_matrix.append(row)
color = Color()
print(color.blue + "Spawning " + color.red + str(num_cores) + color.blue + " processes to make distance matrix." + color.done)
processes = []
for i in range(num_cores):
p = multiprocessing.Process(target=self.distance_matrix_worker, args=(seq_keys, length_threshold, dist_matrix, already_compared, lock, i, gb_dir))
p.start()
processes.append(p)
for p in processes:
p.join()
sys.stdout.write("\n")
sys.stdout.flush()
self.distance_matrix = dist_matrix
def print_data(self):
"""
Prints out details on the final aligned supermatrix.
"""
# TODO: make the output of this more useful
color = Color()
print(color.blue + "Supermatrix attributes:")
records = SeqIO.parse(self.file, "fasta")
num_records = 0
total_missing = 0
for record in records:
otu = record.description
missing = 0
for letter in record.seq:
if letter == '?':
missing += 1
total_missing += 1
print(color.yellow + "OTU: " + color.red + otu + color.yellow +
" % missing data = " + color.red +
str(round(missing / float(len(record.seq)), 2)))
num_records += 1
matrix_length = len(record.seq)
print(color.blue + "Total number of OTUs = " + color.red +
str(num_records))
print(color.blue + "Total length of matrix = " + color.red +
str(matrix_length))
print(color.blue + "Taxon coverage density = " + color.red +
str(self.get_coverage_density()))
print(color.blue + "Total % missing data = " + color.red +
str(round(total_missing /
float(matrix_length * num_records), 2)) + color.done)
def __init__(self, world: 'World', pos: Tuple[int, int], size: Tuple[int,
int]):
super().__init__(world)
self.pos = pos
self.size = size
self.color = Color(80, 80, 80)
def print_PD(self):
"""
Prints partial decisiveness.
"""
color = Color()
print(color.blue + "Partial decisiveness (fraction of triples) = " +
color.red + str(self.get_PD()) + color.done)
def print_data(self):
"""
Prints the name of each DNA region, the number of taxa, the aligned length,
missing data (%), and taxon coverage density
"""
# first get list of all taxa
taxa = self.get_all_taxa()
# print data for each region
i = 1
color = Color()
for alignment in self.files:
records = list(SeqIO.parse(alignment, "fasta"))
if self.user_provided:
region_name = alignment
else:
descriptors = records[0].description.split(" ")
region_name = " ".join(descriptors[5:])
print(color.blue + "Aligned cluster #: " + color.red + str(i) +
color.done)
print(color.yellow + "DNA region: " + color.red + region_name +
color.done)
print(color.yellow + "OTUs: " + color.red + str(len(records)) +
color.done)
print(color.yellow + "Aligned length: " + color.red +
str(len(records[0].seq)) + color.done)
print(color.yellow + "Missing data (%): " + color.red +
str(round(100 -
(100 * len(records) / float(len(taxa))), 1)) +
color.done)
print(color.yellow + "Taxon coverage density: " + color.red +
str(round(len(records) / float(len(taxa)), 2)) + color.done)
i += 1
def align_cluster(self, cluster_file):
"""
Worker fuction for align_clusters
Inputs a FASTA file containing an unaligned sequence cluster.
Uses MAFFT to align the cluster.
"""
mafft_cline = MafftCommandline(input=cluster_file)
mafft_cline.set_parameter("--auto", True)
mafft_cline.set_parameter("--adjustdirection", True)
color = Color()
print(color.red + str(mafft_cline) + color.done)
sys.stdout.flush()
if cluster_file.find("/") != -1:
alignment_file = "alignments" + cluster_file[cluster_file.index("/"
):]
else:
alignment_file = "alignments/" + cluster_file
try:
stdout, stderr = mafft_cline()
with open(alignment_file, "w") as handle:
handle.write(stdout)
except:
print(
color.red +
"Error: alignment file not generated. Please check your MAFFT installation."
+ color.done)
return alignment_file
def print_data(self):
color = Color()
print(color.blue + "Name = " + color.red + self.name)
print(color.blue + "Sequence = " + color.red + self.sequence)
print(color.blue + "Accessions = " + color.red)
print(self.accessions)
print(color.blue + "Sequence_lengths = " + color.red)
print(self.sequence_lengths)
def state(self):
""" Exec state of this component. """
rect = self.root.find_element_by_css_selector('rect')
style = Style(rect.get_attribute('style'))
stroke = Color.from_string(style.stroke)
# red stroke
if (stroke == Color(255, 0, 0)):
return 'INVALID'
# green stroke
if (stroke == Color(0, 255, 0)):
return 'VALID'
# blue stroke
if (stroke == Color(0, 0, 255)):
return 'RUNNING'
return 'UNKNOWN'
def __init__(self, window: Window, source: WindowHandler):
super().__init__(window)
self.source = source
self.levels = self._init_levels()
self.level = self.levels[0]
self.player = Player(self)
self.window = window
self.text_font = Font('monospace', 16, window.hidpi_factor)
self.text_font_color = Color(255, 255, 255)
def print_search_status(self, i, total):
color = Color()
sys.stdout.write('\r' + color.yellow + 'Ingroup sequences found: ' \
+ color.red + str(len(self.ingroup_keys)) + color.yellow \
+ ' Outgroup sequences found: ' + color.red \
+ str(len(self.outgroup_keys)) + color.yellow \
+ ' Percent searched: ' + color.red \
+ str(round( 100 * float(i) / total , 1)) + color.done
)
sys.stdout.flush()
def draw_text(
self,
id: str = None,
x: float = 0,
y: float = 0,
w: float = -1.0, # By default, DON'T restrict w/h
h: float = -1.0,
text: str = "",
color: Color = BLACK,
font_name: str = "Ubuntu",
font_size: int = 16,
align: str = "left",
line_spacing: int = 0,
justify: bool = False,
debug: bool = False,
):
"""Draw the configured text widget on the canvas.
"""
# First, draw a debug box (if requested)
if debug:
self.draw_rect(x=x,
y=y,
w=w,
h=h,
color=Color(0.0, 1.0, 1.0, 1.0),
stroke=True,
fill=False)
# Process the inputs
text = text.replace("\\n", "\n")
alignment = {
"left": TextAlignment.Left,
"center": TextAlignment.Center,
"right": TextAlignment.Right,
}[align] # TODO: Validate on the parser
# Configure the text
self.renderer.set_font(font_name, font_size)
self.renderer.configure_text_layout(
width=w,
height=h,
line_spacing=line_spacing,
alignment=alignment,
justify=justify,
)
self.renderer.set_color(*color)
self.renderer.set_text(text)
# Draw the text
with self.renderer.translate(x, y):
self.renderer.paint_text()
def search(self, gb, max_ingroup):
"""
Perform search of all GB SeqRecords for ingroup/outgroup,
and save results of search to file.
"""
keys = gb.keys()
total = len(keys)
i = 0
ingroup_terms = self.ingroup.split('+')
outgroup_terms = self.outgroup.split('+')
for key in keys:
found = False
try:
gb_tax = gb[key].annotations['taxonomy']
if type(gb_tax) == str:
taxonomy = set([gb_tax])
else:
taxonomy = set(gb_tax)
gb_org = gb[key].annotations['organism']
if type(gb_org) == str:
organism = set([gb_org])
else:
organism = set(gb_org)
tax_and_org = taxonomy.union(organism)
for term in ingroup_terms:
if term in tax_and_org:
self.ingroup_keys.append(key)
found = True
break
if not found:
for term in outgroup_terms:
if term in tax_and_org:
self.outgroup_keys.append(key)
break
except:
sys.stdout.write('\n')
sys.stdout.flush()
color = Color()
print(color.red + 'Caught Biopython GenBank parsing error! NCBI accession: ' + color.yellow + key + color.done)
self.print_search_status(i, total)
i += 1
if max_ingroup is not None and len(self.ingroup_keys) == max_ingroup:
sys.stdout.write('\n')
sys.stdout.flush()
self.write_file()
return
sys.stdout.write('\n')
sys.stdout.flush()
self.write_file()
def get_boid_color(chromatid_a, chromatid_b):
colors_1 = get_color_genes(chromatid_a)
colors_2 = get_color_genes(chromatid_b)
def average_color(a, b):
final_color = int((a + b)/2)
if final_color > 255:
final_color = 255
if final_color < 0:
final_color = 0
return final_color
R, G, B = [average_color(colors_1[i], colors_2[i]) for i in range(3)]
return Color(R, G, B)
def read_file(self):
"""
Loads results of GB search from file.
"""
groups = pickle.load( open( "gb_search_results", "rb" ) )
self.ingroup = groups["ingroup"]
self.outgroup = groups["outgroup"]
self.ingroup_keys = groups["ingroup_keys"]
self.outgroup_keys = groups["outgroup_keys"]
color = Color()
print(color.yellow + 'This search was already performed. Loading previous results...' + color.done)
print(color.yellow + 'Ingroup sequences found: ' \
+ color.red + str(len(self.ingroup_keys)) + color.yellow \
+ ' Outgroup sequences found: ' + color.red \
+ str(len(self.outgroup_keys)) + color.done \
)
def _parse_expression(key: str, exp: str):
"""Take in an expression and safely transform it into a python expression.
"""
# Do some cleaning
exp = exp.strip()
# TODO: Use regex to validate first?
# TODO: Maybe PyParsing could validate it
# TODO: Support expressions
if key in [
"x", "y", "w", "h", "radius", "padding_x", "padding_y",
"line_spacing"
]:
return float(exp)
if key == "color":
return Color(*tuple(float(e) for e in exp.split(',')))
return exp
def render(self, canvas: simplegui.Canvas):
bounds = self.get_bounds()
dpi_factor = self.window.hidpi_factor
# Draw player.
if PLAYER_POTATO:
dest_center = self.pos + self.size / 2
index = (self.roll // self.sprite_cols) % self.sprite_cols
self.sprite.draw(canvas, dest_center * dpi_factor,
self.size * dpi_factor, (index, 0))
else:
point_list = [p.multiply(dpi_factor).into_tuple() for p in bounds]
color = Color(120, 120, 200)
canvas.draw_polygon(point_list, 1, str(color), str(color))
# Update position.
self.last_pos = self.pos.copy()
self.pos.add(self.vel)
self.roll += self.vel.x * 2 / PLAYER_VELOCITY[0]
self.roll += 1
if abs(self.vel.x) > PLAYER_VELOCITY[0]:
self.vel.x = math.copysign(PLAYER_VELOCITY[0], self.vel.x)
# Check collisions position.
self.on_ground = False
bounds = self.get_bounds()
if not self.is_dying:
for item in self.world.level.items:
if item.collides_with(bounds):
item.on_collide(self)
self.vel.add(self.accel)
# Do gravity and platform collision.
if self.on_ground:
self.vel.y = 0
self.accel.y = 0
else:
self.accel.y = -ACCEL_GRAVITY
if bounds.max.y >= WINDOW_SIZE[1] or bounds.min.x <= 0:
self.on_death()
def download(cls, divisions, path):
"""
Downloads and uncompresses files for a GenBank division.
Path should be the absolute path to save the GB files.
"""
for division_input in divisions:
color = Color()
division = str(division_input).lower()
print(color.purple + "Connecting to ftp.ncbi.nlm.nih.gov..." + color.done)
ftp = FTP("ftp.ncbi.nlm.nih.gov")
ftp.login()
print(color.yellow + "Opening directory genbank..." + color.done)
ftp.cwd("genbank")
file_list = ftp.nlst()
i = 1
file_name = "gb" + division + str(i) + ".seq.gz"
if not os.path.exists(path):
os.makedirs(path)
path = path + "/"
while file_name in file_list:
print(color.red + "Downloading file " + file_name + color.done)
file = open(path + file_name, "wb")
cls.getbinary(ftp, file_name, file)
file.close()
print(color.yellow + "Uncompressing file " + file_name + color.done)
file = gzip.open(path + file_name, "rb")
file_content = file.read()
file.close()
file = open(path + file_name[:-3], "wb")
file.write(file_content)
file.close()
os.remove(path + file_name)
i += 1
file_name = 'gb' + division + str(i) + '.seq.gz'
# check if any files were downloaded
if i == 1:
print(color.red + "GenBank division " + division_input \
+ " not found. Please use a valid division name " \
+ "(e.g. VRT, INV, PLN)." + color.done
)
sys.exit(0)
ftp.quit()
def sqlite(path):
"""
Sets up the SQLite db for the GenBank division.
Path is the absolute path of the GB files.
Returns a dictionary of SeqRecord objects.
"""
color = Color()
if os.path.exists(path + "/gb.idx"):
print(color.purple + "Genbank database already downloaded. Indexing sequences..." + color.done)
return SeqIO.index_db(path + "/gb.idx")
else:
files = os.listdir(path)
path_files = []
if len(files) == 0:
print(color.red + "GenBank files not found. Re-download with the -d option. See --help for more details." + color.done)
sys.exit(0)
for file in files:
path_files.append(path + "/" + file)
print(color.purple + "Genbank database already downloaded. Indexing sequences..." + color.done)
return SeqIO.index_db(path + "/gb.idx", path_files, "genbank")
def __init__(self, cluster_files, aligned, num_cores):
"""
Input parameters:
cluster_files: a list of FASTA files
aligned: a string that indicated whether each file contains an
unaligned sequence cluster or one already aligned.
Creates new processes to align each sequence cluster.
Generates a list of aligned FASTA files.
"""
taxa = None
alignment_files = []
if not os.path.exists("alignments"):
os.makedirs("alignments")
color = Color()
if aligned == "unaligned":
self.user_provided = False
self.sumac_aligned = False
print(color.blue + "Spawning " + color.red + str(num_cores) +
color.blue + " processes to align clusters." + color.done)
pool = multiprocessing.Pool(num_cores)
alignment_files = pool.map(self.align_cluster, cluster_files)
pool.close()
pool.join()
self.files = alignment_files
if (not os.path.isfile(alignment_files[0])) or os.path.getsize(
alignment_files[0]) == 0:
print(color.red + "Error: MAFFT is not installed correctly." +
color.done)
sys.exit()
elif aligned == "aligned":
print(color.purple + "Loading user-provided alignments..." +
color.done)
self.user_provided = True
self.sumac_aligned = False
self.files = cluster_files
else:
print(color.purple + "Loading SUMAC alignments..." + color.done)
self.user_provided = False
self.sumac_aligned = True
self.files = cluster_files
def calculate_PD(self):
"""
Method to calculate the fraction of triples, a measure of partial decisiveness (PD).
See: Sanderson, M.J., McMahon, M.M. & Steel, M., 2010. BMC evolutionary biology, 10.
"""
color = Color()
decisive_triples = 0
total_triples = 0
total = self.binomial_coefficient(len(self.otus), 3)
i = 0
# nested loops to run through every possible triplet
for otu1 in self.otus:
if i < (len(self.otus) - 2):
j = 0
for otu2 in self.otus:
if i < j:
k = 0
for otu3 in self.otus:
if j < k:
sys.stdout.write("\r" + color.blue + "Calculating PD: " + color.red + str(round(100 * total_triples/float(total), 4)) + \
"% " + color.blue + "finished" + color.done)
sys.stdout.flush()
# do PD calculations for this triplet
triplet = [otu1, otu2, otu3]
decisive, decisive_loci = self.calculate_triplet_PD(
triplet)
decisive_triples += decisive
total_triples += 1
# triplet calculations for OTU and loci decisiveness scores
self.update_OTU_decisiveness(triplet, decisive)
self.update_locus_decisiveness(
decisive_loci, decisive)
k += 1
j += 1
i += 1
sys.stdout.write("\r" + color.blue + "Calculating PD: " + color.red +
"100.00% " + color.blue + "finished\n" + color.done)
sys.stdout.flush()
return round(decisive_triples / float(total_triples), 2)
def get_border_color(self) -> Color:
"""
Returns the border color of the rectangle.
"""
return Color(200, 200, 200)
def get_fill_color(self) -> Color:
"""
Returns the fill color of the rectangle.
"""
return Color(0, 0, 0)
def main():
# parse the command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("--download_gb", "-d", help="Name of the GenBank division to download (e.g. PLN or MAM).")
parser.add_argument("--download_gb2", "-d2", help="""Name of the optional second GenBank division to download. Use this
option if the ingroup and outgroup are in different GenBank divisions.""")
parser.add_argument("--path", "-p", help="Absolute path to download GenBank files to. Defaults to ./genbank/")
parser.add_argument("--ingroup", "-i", help="Ingroup clade to build supermatrix.")
parser.add_argument("--outgroup", "-o", help="Outgroup clade to build supermatrix.")
parser.add_argument("--cores", "-c", help="The number of CPU cores to use for parallel processing. Defaults to the max available.")
parser.add_argument("--id", "-id", help="UCLUST id threshold to cluster taxa. Defaults to 0.50")
parser.add_argument("--evalue", "-e", help="BLAST E-value threshold to cluster taxa. Defaults to 1e-10")
parser.add_argument("--length", "-l", help="Threshold of sequence length percent similarity to cluster taxa. Defaults to 0.25")
parser.add_argument("--maxlength", "-maxl", help="Maximum length of sequences to include in UCLUST clusters. Defaults to 5000")
parser.add_argument("--minlength", "-minl", help="Minimum length of sequences to include in UCLUST clusters. Defaults to 100")
parser.add_argument("--min_clusters", "-minc", help="Minimum number of taxa needed for clusters. Defaults to 4")
parser.add_argument("--max_ingroup", "-m", help="Maximum number of taxa to include in ingroup. Default is none (no maximum limit).")
parser.add_argument("--guide", "-g", help="""FASTA file containing sequences to guide cluster construction. If this option is
selected then all-by-all BLAST comparisons are not performed.""")
parser.add_argument("--alignments", "-a", nargs='+', help="List of aligned FASTA files to build supermatrix instead of mining GenBank.")
parser.add_argument("--salignments", "-sa", nargs='+', help="List of SUMAC alignments to build supermatrix instead of mining GenBank.")
parser.add_argument("--search", "-s", action='store_true', help="Turn on search and cluster mode. Will not make alignments or supermatrix.")
parser.add_argument("--decisiveness", "-de", action='store_true', help="Calculate partial decisiveness. For larger matrices this may be slow.")
parser.add_argument("--hac", action='store_true', help="Use HAC single-linkage clustering algorithm instead of the default UCLUST algorithm.")
parser.add_argument("--slink", action='store_true', help="Use the SLINK clustering algorithm instead of the default UCLUST algorithm.")
args = parser.parse_args()
sys.stdout = Logger()
color = Color()
print("")
print(color.blue + "SUMAC: supermatrix constructor v2.22" + color.done)
print("")
num_cores = multiprocessing.cpu_count()
if args.cores and int(args.cores) <= num_cores:
num_cores = int(args.cores)
if args.alignments:
# if the user provides alignments:
alignment_files = args.alignments
alignments = Alignments(alignment_files, "aligned", num_cores)
elif args.salignments:
# if the user inputs SUMAC alignments from previous run
alignment_files = args.salignments
alignments = Alignments(alignment_files, "sumac_aligned", num_cores)
else:
if args.search:
print(color.yellow + "Running in search and cluster mode. Clusters will not be aligned and supermatrix will not assembled." + color.done)
# first download and set up sqllite db if necessary
if args.path:
gb_dir = args.path
else:
gb_dir = os.path.abspath("genbank/")
# if the user requests downloading
if args.download_gb:
divisions = [args.download_gb]
if args.download_gb2:
divisions.append(args.download_gb2)
GenBankSetup.download(divisions, gb_dir)
print(color.yellow + "Setting up SQLite database..." + color.done)
gb = GenBankSetup.sqlite(gb_dir)
# the user didn't request downloading, so check for genbank directory
elif not os.path.exists(gb_dir):
print(color.red + "GenBank database not downloaded. Re-run with the -d option. See --help for more details." + color.done)
sys.exit(0)
# the genbank directory exists so check for sequences and index them
else:
gb = GenBankSetup.sqlite(gb_dir)
print(color.purple + "%i sequences indexed!" % len(gb) + color.done)
# check for ingroup and outgroup
if args.ingroup:
ingroup = args.ingroup
if args.outgroup:
outgroup = args.outgroup
else:
outgroup = "NONE"
else:
print(color.red + "Please specify ingroup. See --help for details." + color.done)
sys.exit(0)
# search db for ingroup and outgroup sequences
print(color.blue + "Ingroup = " + ingroup + color.done)
if args.outgroup:
print(color.blue + "Outgroup = " + outgroup + color.done)
print(color.blue + "Searching for ingroup and outgroup sequences..." + color.done)
if args.max_ingroup:
search_results = GenBankSearch(gb, ingroup, outgroup, int(args.max_ingroup))
else:
search_results = GenBankSearch(gb, ingroup, outgroup)
ingroup_keys = search_results.ingroup_keys
outgroup_keys = search_results.outgroup_keys
all_seq_keys = ingroup_keys + outgroup_keys
if len(all_seq_keys) == 0:
print(color.red + "No sequences found for the ingroup and outgroup!" + color.done)
sys.exit(0)
# determine sequence length similarity threshold
length_threshold = 0.25
if args.length:
length_threshold = float(args.length)
print(color.blue + "Using sequence length similarity threshold " + color.red + str(length_threshold) + color.done)
# determine e-value threshold
id_threshold = 0.5
if args.id:
id_threshold = float(args.id)
print(color.blue + "Using UCLUST id threshold " + color.red + str(id_threshold) + color.done)
# determine e-value threshold
evalue_threshold = (1.0/10**10)
if args.evalue:
evalue_threshold = float(args.evalue)
print(color.blue + "Using BLAST e-value threshold " + color.red + str(evalue_threshold) + color.done)
# now build clusters, first checking whether we are using FASTA file of guide sequences
# or doing all-by-all comparisons
if args.guide:
# use FASTA file of guide sequences
print(color.blue + "Building clusters using the guide sequences..." + color.done)
cluster_builder = GuidedClusterBuilder(args.guide, all_seq_keys, length_threshold, evalue_threshold, gb_dir, num_cores)
else:
# cluster using UCLUST
uclust_error = False
if not (args.slink or args.hac):
print(color.blue + "Clustering sequences with UCLUST...")
maxlength = 5000
minlength = 100
if args.maxlength:
maxlength = int(args.maxlength)
if args.minlength:
minlength = int(args.minlength)
cluster_builder = UCLUSTClusterBuilder(gb, all_seq_keys, gb_dir, num_cores, minlength, maxlength, length_threshold, id_threshold, evalue_threshold)
if (cluster_builder.error == True):
uclust_error = True
else:
print(color.purple + "Clustering completed..." + color.done)
if (args.slink or args.hac) or (uclust_error == True):
# make distance matrix
print(color.blue + "Making distance matrix for all sequences..." + color.done)
distance_matrix = DistanceMatrixBuilder(gb, all_seq_keys, length_threshold, gb_dir, num_cores).distance_matrix
# cluster sequences
if args.hac:
print(color.purple + "Clustering sequences using the HAC algorithm..." + color.done)
cluster_builder = HACClusterBuilder(all_seq_keys, distance_matrix, evalue_threshold)
else:
print(color.purple + "Clustering sequences using the SLINK algorithm..." + color.done)
cluster_builder = SLINKClusterBuilder(all_seq_keys, distance_matrix, evalue_threshold)
print(color.purple + "Found " + color.red + str(len(cluster_builder.clusters)) + color.purple + " clusters." + color.done)
if len(cluster_builder.clusters) == 0:
print(color.red + "No clusters found." + color.done)
sys.exit(0)
# filter clusters, make FASTA files
print(color.yellow + "Building sequence matrices for each cluster." + color.done)
min_clusters = 4
if args.min_clusters:
min_clusters = int(args.min_clusters)
if (args.slink or args.hac or args.guide) or (uclust_error == True):
cluster_builder.assemble_fasta(gb, min_clusters)
else:
cluster_builder.assemble_fasta_uclust(min_clusters)
print(color.purple + "Kept " + color.red + str(len(cluster_builder.clusters)) + color.purple + " clusters, discarded those with < " + str(min_clusters) + " taxa." + color.done)
# if we are in search and cluster mode we are done
if args.search:
sys.exit(0)
if len(cluster_builder.clusters) == 0:
print(color.red + "No clusters left to align." + color.done)
sys.exit(0)
# now align each cluster with MAFFT
print(color.blue + "Aligning clusters with MAFFT..." + color.done)
alignments = Alignments(cluster_builder.cluster_files, "unaligned", num_cores)
alignments.print_data()
alignments.make_gene_region_csv()
# concatenate alignments
print(color.purple + "Concatenating alignments..." + color.done)
supermatrix = Supermatrix(alignments)
try:
imp.find_module('matplotlib')
imp.find_module('numpy')
matplot = True
except ImportError:
matplot = False
print(color.red + "Skipping generating graphs since matplotlib is not installed." + color.done)
if not args.alignments: # and not args.salignments:
# only make genbank_csv if the sequences were mined direct from genbank
supermatrix.make_genbank_csv()
supermatrix.print_data()
if matplot:
supermatrix.make_sequence_data_figure()
if args.decisiveness:
supermatrix.print_PD()
if matplot:
supermatrix.make_sequence_decisiveness_figure()
supermatrix.make_decisiveness_csv()
print(color.yellow + "Final supermatrix: " + color.red + "alignments/supermatrix_concatenated.fasta" + color.done)
def distance_matrix_worker(self, seq_keys, length_threshold, dist_matrix, already_compared, lock, process_num, gb_dir):
"""
Worker process for make_distance_matrix(). Takes a list "already_compared" of sequences that have
already had all pairwise comparisons. Each worker process will work making pairwise comparisons
for a different sequence, adding them to the "already_compared" list as they are completed.
"""
# each process must load its own sqlite gb
gb = SeqIO.index_db(gb_dir + "/gb.idx")
process_num = str(process_num)
i = 0
color = Color()
for key in seq_keys:
# check whether another process is already comparing this row
compare_row = False
with lock:
if key not in already_compared:
already_compared.append(key)
compare_row = True
if compare_row:
# make the blast query
record1 = gb[key]
output_handle = open('query' + process_num + '.fasta', 'w')
SeqIO.write(record1, output_handle, 'fasta')
output_handle.close()
# make blast database
j = 0
output_handle = open('blast_db' + process_num + '.fasta', 'w')
records = []
for key2 in seq_keys:
# only add sequences that have not yet been compared
if j > i:
record = gb[key2]
records.append(record)
if j == i:
row = dist_matrix[i]
row[j] = 0.0
dist_matrix[i] = row
j += 1
SeqIO.write(records, output_handle, 'fasta')
output_handle.close()
if len(records) > 0:
# blast query against blast_db
blastn_cmd = NcbiblastnCommandline(query='query' + process_num + '.fasta', subject='blast_db' + process_num + '.fasta', \
out='blast' + process_num + '.xml', outfmt=5)
stdout, stderr = blastn_cmd()
# parse blast output
j = i + 1
blastn_xml = open('blast' + process_num + '.xml', 'r')
blast_records = NCBIXML.parse(blastn_xml)
for blast_record in blast_records:
for alignment in blast_record.alignments:
# loop through each high-scoring segment pair (HSP)
for hsp in alignment.hsps:
length1 = len(record1.seq)
length2 = alignment.length
# first check if length similarity threshold met
if (length1 < length2 * (1 + float(length_threshold))) and (length1 > length2 * (1 - float(length_threshold))):
# blast hit found, set distance to e-value
row = dist_matrix[i]
row[j] = hsp.expect
dist_matrix[i] = row
row = dist_matrix[j]
row[i] = hsp.expect
dist_matrix[j] = row
else:
# set distance to 50.0 if length similarity threshold not met
row = dist_matrix[i]
row[j] = 50.0
dist_matrix[i] = row
row = dist_matrix[j]
row[i] = 50.0
dist_matrix[j] = row
j += 1
blastn_xml.close()
i += 1
# update status
percent = str(round(100 * len(already_compared)/float(len(seq_keys)), 2))
sys.stdout.write('\r' + color.blue + 'Completed: ' + color.red + str(len(already_compared)) + '/' + str(len(seq_keys)) + ' (' + percent + '%)' + color.done)
sys.stdout.flush()
# done looping through all keys, now clean up
if os.path.exists('blast_db' + process_num + '.fasta'):
os.remove('blast_db' + process_num + '.fasta')
if os.path.exists("blast" + process_num + ".xml"):
os.remove("blast" + process_num + ".xml")
if os.path.exists("query" + process_num + ".fasta"):
os.remove("query" + process_num + ".fasta")
if os.path.exists("subject" + process_num + ".fasta"):
os.remove("subject" + process_num + ".fasta")
def distance_matrix_worker_old(self, seq_keys, length_threshold, dist_matrix, already_compared, lock, process_num, gb_dir):
"""
Worker process for make_distance_matrix(). Takes a list "already_compared" of sequences that have
already had all pairwise comparisons. Each worker process will work making pairwise comparisons
for a different sequence, adding them to the "already_compared" list as they are completed.
"""
# each process must load its own sqlite gb
gb = SeqIO.index_db(gb_dir + "/gb.idx")
process_num = str(process_num)
i = 0
color = Color()
for key in seq_keys:
# check whether another process is already comparing this row
compare_row = False
with lock:
if key not in already_compared:
already_compared.append(key)
compare_row = True
if compare_row:
# get the sequence record to compare
record1 = gb[key]
output_handle = open('query' + process_num + '.fasta', 'w')
SeqIO.write(record1, output_handle, 'fasta')
output_handle.close()
j = 0
for key2 in seq_keys:
# only calculate e-values for pairs that have not yet been compared
if dist_matrix[i][j] == 99:
if key == key2:
row = dist_matrix[i]
row[j] = 0.0
dist_matrix[i] = row
# check sequence lengths
else:
# print("proc # = "+process_num+" i = "+str(i)+ " j = "+str(j))
record2 = gb[key2]
length1 = len(record1.seq)
length2 = len(record2.seq)
# set distance to 50.0 if length similarity threshold not met
if (length2 < length1 * (1 + float(length_threshold))) and (length2 > length1 * (1 - float(length_threshold))):
row = dist_matrix[i]
row[j] = 50.0
dist_matrix[i] = row
row = dist_matrix[j]
row[i] = 50.0
dist_matrix[j] = row
else:
# do the blast comparison
output_handle = open('query' + process_num + '.fasta', 'w')
SeqIO.write(record2, output_handle, 'fasta')
output_handle.close()
blastn_cmd = NcbiblastnCommandline(query='query' + process_num + '.fasta', subject='subject' + process_num + \
'.fasta', out='blast' + process_num + '.xml', outfmt=5)
stdout, stderr = blastn_cmd()
blastn_xml = open('blast' + process_num + '.xml', 'r')
blast_records = NCBIXML.parse(blastn_xml)
for blast_record in blast_records:
if blast_record.alignments:
if blast_record.alignments[0].hsps:
# blast hit found, set distance to e-value
row = dist_matrix[i]
row[j] = blast_record.alignments[0].hsps[0].expect
dist_matrix[i] = row
row = dist_matrix[j]
row[i] = blast_record.alignments[0].hsps[0].expect
dist_matrix[j] = row
else:
# no blast hit found, set distance to default 10.0
row = dist_matrix[i]
row[j] = 10.0
dist_matrix[i] = row
row = dist_matrix[j]
row[i] = 10.0
dist_matrix[j] = row
blastn_xml.close()
j += 1
i += 1
# update status
percent = str(round(100 * len(already_compared)/float(len(seq_keys)), 2))
sys.stdout.write('\r' + color.blue + 'Completed: ' + color.red + str(len(already_compared)) + '/' + str(len(seq_keys)) + ' (' + percent + '%)' + color.done)
sys.stdout.flush()
# done looping through all keys, now clean up
os.remove("blast" + process_num + ".xml")
os.remove("query" + process_num + ".fasta")
os.remove("subject" + process_num + ".fasta")