def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
md, mh, _ = parse_mapping_file(open(mapping_fp))
body_sites = ['Gut','Tongue','Palm','Forehead']
intraindividual_distances = []
print "Unweighted UniFrac"
for b in body_sites:
dm_fp = "/Users/caporaso/analysis/student-microbiome-project/beta-diversity/unweighted_unifrac_dm.%s_ts_only.txt.gz" % b.lower()
h, d = parse_distmat(qiime_open(dm_fp))
intraindividual_distances.append(get_grouped_distances(h, d, mh, md, 'PersonalID'))
for i in range(len(body_sites)):
for j in range(i):
r = correlated_variability(intraindividual_distances[i],intraindividual_distances[j])
print "%s/%s (n=%d): rho:%1.3f, p=%f" % (body_sites[i],body_sites[j],r[0],r[1][0],r[1][3])
intraindividual_distances = []
print "**"
print "Weighted UniFrac"
for b in body_sites:
dm_fp = "/Users/caporaso/analysis/student-microbiome-project/beta-diversity/weighted_unifrac_dm.%s_ts_only.txt.gz" % b.lower()
h, d = parse_distmat(qiime_open(dm_fp))
intraindividual_distances.append(get_grouped_distances(h, d, mh, md, 'PersonalID'))
for i in range(len(body_sites)):
for j in range(i):
r = correlated_variability(intraindividual_distances[i],intraindividual_distances[j])
print "%s/%s (n=%d): rho:%1.3f, p=%f" % (body_sites[i],body_sites[j],r[0],r[1][0],r[1][3])
def test_get_grouped_distances_between(self):
"""get_grouped_distances() should return a list of between distance
groupings."""
groupings = get_grouped_distances(
self.dist_matrix_header, self.dist_matrix, self.mapping_header, self.mapping, self.field, within=False
)
expected = [
(
"Control",
"Fast",
[
0.72899999999999998,
0.80000000000000004,
0.72099999999999997,
0.76500000000000001,
0.77600000000000002,
0.74399999999999999,
0.749,
0.67700000000000005,
0.73399999999999999,
0.77700000000000002,
0.73299999999999998,
0.72399999999999998,
0.69599999999999995,
0.67500000000000004,
0.65400000000000003,
0.69599999999999995,
0.73099999999999998,
0.75800000000000001,
0.73799999999999999,
0.73699999999999999,
],
)
]
self.assertEqual(groupings, expected)
def test_get_grouped_distances_within(self):
"""get_grouped_distances() should return a list of within distance
groupings."""
groupings = get_grouped_distances(self.dist_matrix_header,
self.dist_matrix, self.mapping_header, self.mapping,
self.field, within=True)
expected = [
('Control', 'Control', [0.625, 0.623, 0.60999999999999999, \
0.57699999999999996, 0.61499999999999999, \
0.64200000000000002, 0.67300000000000004, \
0.68200000000000005, 0.73699999999999999, \
0.70399999999999996]),
('Fast', 'Fast', [0.71799999999999997, 0.66600000000000004, \
0.72699999999999998, 0.59999999999999998, \
0.57799999999999996, 0.623])]
self.assertEqual(groupings, expected)
def test_get_grouped_distances_within(self):
"""get_grouped_distances() should return a list of within distance
groupings."""
groupings = get_grouped_distances(self.dist_matrix_header,
self.dist_matrix, self.mapping_header, self.mapping,
self.field, within=True)
expected = [
('Control', 'Control', [0.625, 0.623, 0.60999999999999999, \
0.57699999999999996, 0.61499999999999999, \
0.64200000000000002, 0.67300000000000004, \
0.68200000000000005, 0.73699999999999999, \
0.70399999999999996]),
('Fast', 'Fast', [0.71799999999999997, 0.66600000000000004, \
0.72699999999999998, 0.59999999999999998, \
0.57799999999999996, 0.623])]
self.assertEqual(groupings, expected)
def test_get_grouped_distances_between(self):
"""get_grouped_distances() should return a list of between distance
groupings."""
groupings = get_grouped_distances(self.dist_matrix_header,
self.dist_matrix, self.mapping_header, self.mapping,
self.field, within=False)
expected = [
('Control', 'Fast', [0.72899999999999998, 0.80000000000000004, \
0.72099999999999997, 0.76500000000000001, \
0.77600000000000002, 0.74399999999999999, \
0.749, 0.67700000000000005, \
0.73399999999999999, 0.77700000000000002, \
0.73299999999999998, 0.72399999999999998, \
0.69599999999999995, 0.67500000000000004, \
0.65400000000000003, 0.69599999999999995, \
0.73099999999999998, 0.75800000000000001, \
0.73799999999999999, 0.73699999999999999])]
self.assertEqual(groupings, expected)
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
# Create the output dir if it doesn't already exist.
try:
create_dir(opts.output_dir)
except:
option_parser.error("Could not create or access output directory "
"specified with the -o option.")
# Parse the distance matrix and mapping file.
try:
dist_matrix_header, dist_matrix = parse_distmat(
open(opts.distance_matrix_fp, 'U'))
except:
option_parser.error(
"This does not look like a valid distance matrix "
"file. Please supply a valid distance matrix file using the -d "
"option.")
try:
mapping, mapping_header, mapping_comments = parse_mapping_file(
open(opts.mapping_fp, 'U'))
except QiimeParseError:
option_parser.error(
"This does not look like a valid metadata mapping "
"file. Please supply a valid mapping file using the -m option.")
fields = opts.fields
fields = map(strip, fields.split(','))
fields = [field.strip('"').strip("'") for field in fields]
if fields is None:
option_parser.error("You must provide at least one field using the -f "
"option.")
# Make sure each field is in the mapping file.
for field in fields:
if field not in mapping_header:
option_parser.error(
"The field '%s' is not in the provided "
"mapping file. Please supply correct fields (using the -f "
"option) corresponding to fields in the mapping file." % field)
# Make sure the y_min and y_max options make sense, as they can be either
# 'auto' or a number.
y_min = opts.y_min
y_max = opts.y_max
try:
y_min = float(y_min)
except ValueError:
if y_min == 'auto':
y_min = None
else:
option_parser.error("The --y_min option must be either a number "
"or 'auto'.")
try:
y_max = float(y_max)
except ValueError:
if y_max == 'auto':
y_max = None
else:
option_parser.error("The --y_max option must be either a number "
"or 'auto'.")
# Generate the various boxplots, depending on what the user wanted
# suppressed. Add them all to one encompassing plot.
for field in fields:
plot_data = []
plot_labels = []
if not opts.suppress_all_within:
plot_data.append(
get_all_grouped_distances(dist_matrix_header,
dist_matrix,
mapping_header,
mapping,
field,
within=True))
plot_labels.append("All within %s" % field)
if not opts.suppress_all_between:
plot_data.append(
get_all_grouped_distances(dist_matrix_header,
dist_matrix,
mapping_header,
mapping,
field,
within=False))
plot_labels.append("All between %s" % field)
if not opts.suppress_individual_within:
within_dists = get_grouped_distances(dist_matrix_header,
dist_matrix,
mapping_header,
mapping,
field,
within=True)
for grouping in within_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
if not opts.suppress_individual_between:
between_dists = get_grouped_distances(dist_matrix_header,
dist_matrix,
mapping_header,
mapping,
field,
within=False)
for grouping in between_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
# We now have our data and labels ready, so plot them!
assert (len(plot_data) == len(plot_labels)), "The number " +\
"of boxplot labels does not match the number of " +\
"boxplots."
if plot_data:
if opts.sort:
# Sort our plot data in order of increasing median.
sorted_data = []
for label, distribution in zip(plot_labels, plot_data):
sorted_data.append(
(label, distribution, median(distribution)))
sorted_data.sort(key=itemgetter(2))
plot_labels = []
plot_data = []
for label, distribution, median_value in sorted_data:
plot_labels.append(label)
plot_data.append(distribution)
width = opts.width
height = opts.height
if width is None:
width = len(plot_data) * opts.box_width + 2
if width <= 0 or height <= 0:
option_parser.error("The specified width and height of the "
"image must be greater than zero.")
plot_figure = generate_box_plots(
plot_data,
x_tick_labels=plot_labels,
title="%s Distances" % field,
x_label="Grouping",
y_label="Distance",
x_tick_labels_orientation='vertical',
y_min=y_min,
y_max=y_max,
whisker_length=opts.whisker_length,
box_width=opts.box_width,
box_color=opts.box_color,
figure_width=width,
figure_height=height)
output_plot_fp = join(opts.output_dir,
"%s_Distances.%s" % (field, opts.imagetype))
plot_figure.savefig(output_plot_fp,
format=opts.imagetype,
transparent=opts.transparent)
else:
option_parser.error("You have chosen to suppress all plots. At "
"least one type of plot must be unsuppressed.")
if not opts.suppress_significance_tests:
sig_tests_f = open(join(opts.output_dir, "%s_Stats.xls" % field),
'w')
sig_tests_results = all_pairs_t_test(
plot_labels,
plot_data,
tail_type=opts.tail_type,
num_permutations=opts.num_permutations)
sig_tests_f.write(sig_tests_results)
sig_tests_f.close()
if opts.save_raw_data:
# Write the raw plot data into a tab-delimited file.
assert (len(plot_labels) == len(plot_data))
raw_data_fp = join(opts.output_dir, "%s_Distances.xls" % field)
raw_data_f = open(raw_data_fp, 'w')
for label, data in zip(plot_labels, plot_data):
raw_data_f.write(label.replace(" ", "_") + "\t")
raw_data_f.write("\t".join(map(str, data)))
raw_data_f.write("\n")
raw_data_f.close()
def make_distance_boxplots(dm_f,
map_f,
fields,
width=None,
height=6.0,
suppress_all_within=False,
suppress_all_between=False,
suppress_individual_within=False,
suppress_individual_between=False,
y_min=0.0,
y_max=1.0,
whisker_length=1.5,
box_width=0.5,
box_color=None,
color_individual_within_by_field=None,
sort=None):
"""Generates various types of boxplots for distance comparisons.
Returns a list of tuples, one for each field. Each tuple contains the
following:
1) the name of the field (string)
2) a matplotlib.figure.Figure object containing the boxplots
3) a list of lists containing the raw plot data that was passed to mpl
4) a list of labels for each of the boxplots (string)
5) a list of mpl-compatible colors (one for each boxplot)
The Figure can be saved, and the raw data and labels can be useful (for
example) performing statistical tests or writing the raw data to disk.
The input arguments are exactly derived from the make_distance_boxplots.py
script (see the script options for details). To avoid duplicated effort,
their descriptions are not reproduced here.
"""
# Parse data files and do some preliminary error checking.
dm_header, dm_data = parse_distmat(dm_f)
map_data, map_header, map_comments = parse_mapping_file(map_f)
if fields is None or len(fields) < 1:
raise ValueError("You must provide at least one field to analyze.")
for field in fields:
if field not in map_header:
raise ValueError("The field '%s' is not in the provided mapping "
"file. Please supply correct fields "
"corresponding to fields in the mapping file." %
field)
# Make sure the y_min and y_max options make sense, as they can be either
# 'auto' or a number.
y_min = _cast_y_axis_extrema(y_min)
y_max = _cast_y_axis_extrema(y_max)
# Collate the distributions of distances that will comprise each boxplot.
# Suppress the generation of the indicated types of boxplots.
results = []
for field in fields:
plot_data = []
plot_labels = []
plot_colors = []
legend = None
# Little bit of duplicate code here... not sure it's worth the effort
# to clean up though.
if not suppress_all_within:
plot_data.append(
get_all_grouped_distances(dm_header,
dm_data,
map_header,
map_data,
field,
within=True))
plot_labels.append("All within %s" % field)
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
if not suppress_all_between:
plot_data.append(
get_all_grouped_distances(dm_header,
dm_data,
map_header,
map_data,
field,
within=False))
plot_labels.append("All between %s" % field)
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
if not suppress_individual_within:
within_dists = get_grouped_distances(dm_header,
dm_data,
map_header,
map_data,
field,
within=True)
field_states = []
for grouping in within_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
field_states.append(grouping[0])
# If we need to color these boxplots by a field, build up a
# list of colors and a legend.
if color_individual_within_by_field is not None:
colors, color_mapping = _color_field_states(
format_mapping_file(map_header,
map_data).split('\n'), dm_header,
field, field_states, color_individual_within_by_field)
plot_colors.extend(colors)
legend = (color_mapping.values(), color_mapping.keys())
else:
plot_colors.extend([box_color] * len(field_states))
if not suppress_individual_between:
between_dists = get_grouped_distances(dm_header,
dm_data,
map_header,
map_data,
field,
within=False)
for grouping in between_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
assert (len(plot_data) == len(plot_labels) and
len(plot_labels) == len(plot_colors)), "The number " +\
"of boxplot labels and colors do not match the number of " +\
"boxplots."
# We now have our data and labels ready, so plot them!
if plot_data:
if sort is not None:
plot_data, plot_labels, plot_colors = _sort_distributions(
plot_data, plot_labels, plot_colors, sort)
if width is None:
width = len(plot_data) * box_width + 2
if width <= 0 or height <= 0:
raise ValueError("The specified width and height of the plot "
"must be greater than zero.")
plot_figure = boxplots(plot_data,
x_tick_labels=plot_labels,
title="%s Distances" % field,
x_label="Grouping",
y_label="Distance",
x_tick_labels_orientation='vertical',
y_min=y_min,
y_max=y_max,
whisker_length=whisker_length,
box_width=box_width,
box_colors=plot_colors,
figure_width=width,
figure_height=height,
legend=legend)
results.append(
(field, plot_figure, plot_data, plot_labels, plot_colors))
else:
raise ValueError("The generation of all plots was suppressed. At "
"least one type of plot must be unsuppressed.")
return results
def make_distance_boxplots(
dm_f,
map_f,
fields,
width=None,
height=6.0,
suppress_all_within=False,
suppress_all_between=False,
suppress_individual_within=False,
suppress_individual_between=False,
y_min=0.0,
y_max=1.0,
whisker_length=1.5,
box_width=0.5,
box_color=None,
color_individual_within_by_field=None,
sort=None,
):
"""Generates various types of boxplots for distance comparisons.
Returns a list of tuples, one for each field. Each tuple contains the
following:
1) the name of the field (string)
2) a matplotlib.figure.Figure object containing the boxplots
3) a list of lists containing the raw plot data that was passed to mpl
4) a list of labels for each of the boxplots (string)
5) a list of mpl-compatible colors (one for each boxplot)
The Figure can be saved, and the raw data and labels can be useful (for
example) performing statistical tests or writing the raw data to disk.
The input arguments are exactly derived from the make_distance_boxplots.py
script (see the script options for details). To avoid duplicated effort,
their descriptions are not reproduced here.
"""
# Parse data files and do some preliminary error checking.
dm_header, dm_data = parse_distmat(dm_f)
map_data, map_header, map_comments = parse_mapping_file(map_f)
if fields is None or len(fields) < 1:
raise ValueError("You must provide at least one field to analyze.")
for field in fields:
if field not in map_header:
raise ValueError(
"The field '%s' is not in the provided mapping "
"file. Please supply correct fields "
"corresponding to fields in the mapping file." % field
)
# Make sure the y_min and y_max options make sense, as they can be either
# 'auto' or a number.
y_min = _cast_y_axis_extrema(y_min)
y_max = _cast_y_axis_extrema(y_max)
# Collate the distributions of distances that will comprise each boxplot.
# Suppress the generation of the indicated types of boxplots.
results = []
for field in fields:
plot_data = []
plot_labels = []
plot_colors = []
legend = None
# Little bit of duplicate code here... not sure it's worth the effort
# to clean up though.
if not suppress_all_within:
plot_data.append(get_all_grouped_distances(dm_header, dm_data, map_header, map_data, field, within=True))
plot_labels.append("All within %s" % field)
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
if not suppress_all_between:
plot_data.append(get_all_grouped_distances(dm_header, dm_data, map_header, map_data, field, within=False))
plot_labels.append("All between %s" % field)
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
if not suppress_individual_within:
within_dists = get_grouped_distances(dm_header, dm_data, map_header, map_data, field, within=True)
field_states = []
for grouping in within_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
field_states.append(grouping[0])
# If we need to color these boxplots by a field, build up a
# list of colors and a legend.
if color_individual_within_by_field is not None:
colors, color_mapping = _color_field_states(
format_mapping_file(map_header, map_data).split("\n"),
dm_header,
field,
field_states,
color_individual_within_by_field,
)
plot_colors.extend(colors)
legend = (color_mapping.values(), color_mapping.keys())
else:
plot_colors.extend([box_color] * len(field_states))
if not suppress_individual_between:
between_dists = get_grouped_distances(dm_header, dm_data, map_header, map_data, field, within=False)
for grouping in between_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
if color_individual_within_by_field is not None:
plot_colors.append(None)
else:
plot_colors.append(box_color)
assert len(plot_data) == len(plot_labels) and len(plot_labels) == len(plot_colors), (
"The number " + "of boxplot labels and colors do not match the number of " + "boxplots."
)
# We now have our data and labels ready, so plot them!
if plot_data:
if sort is not None:
plot_data, plot_labels, plot_colors = _sort_distributions(plot_data, plot_labels, plot_colors, sort)
if width is None:
width = len(plot_data) * box_width + 2
if width <= 0 or height <= 0:
raise ValueError("The specified width and height of the plot " "must be greater than zero.")
plot_figure = boxplots(
plot_data,
x_tick_labels=plot_labels,
title="%s Distances" % field,
x_label="Grouping",
y_label="Distance",
x_tick_labels_orientation="vertical",
y_min=y_min,
y_max=y_max,
whisker_length=whisker_length,
box_width=box_width,
box_colors=plot_colors,
figure_width=width,
figure_height=height,
legend=legend,
)
results.append((field, plot_figure, plot_data, plot_labels, plot_colors))
else:
raise ValueError(
"The generation of all plots was suppressed. At " "least one type of plot must be unsuppressed."
)
return results
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
# Create the output dir if it doesn't already exist.
try:
create_dir(opts.output_dir)
except:
option_parser.error("Could not create or access output directory "
"specified with the -o option.")
# Parse the distance matrix and mapping file.
try:
dist_matrix_header, dist_matrix = parse_distmat(
open(opts.distance_matrix_fp, 'U'))
except:
option_parser.error("This does not look like a valid distance matrix "
"file. Please supply a valid distance matrix file using the -d "
"option.")
try:
mapping, mapping_header, mapping_comments = parse_mapping_file(
open(opts.mapping_fp, 'U'))
except QiimeParseError:
option_parser.error("This does not look like a valid metadata mapping "
"file. Please supply a valid mapping file using the -m option.")
fields = opts.fields
fields = map(strip, fields.split(','))
fields = [field.strip('"').strip("'") for field in fields]
if fields is None:
option_parser.error("You must provide at least one field using the -f "
"option.")
# Make sure each field is in the mapping file.
for field in fields:
if field not in mapping_header:
option_parser.error("The field '%s' is not in the provided "
"mapping file. Please supply correct fields (using the -f "
"option) corresponding to fields in the mapping file."
% field)
# Make sure the y_min and y_max options make sense, as they can be either
# 'auto' or a number.
y_min = opts.y_min
y_max = opts.y_max
try:
y_min = float(y_min)
except ValueError:
if y_min == 'auto':
y_min = None
else:
option_parser.error("The --y_min option must be either a number "
"or 'auto'.")
try:
y_max = float(y_max)
except ValueError:
if y_max == 'auto':
y_max = None
else:
option_parser.error("The --y_max option must be either a number "
"or 'auto'.")
# Generate the various boxplots, depending on what the user wanted
# suppressed. Add them all to one encompassing plot.
for field in fields:
plot_data = []
plot_labels = []
if not opts.suppress_all_within:
plot_data.append(get_all_grouped_distances(dist_matrix_header,
dist_matrix, mapping_header, mapping, field, within=True))
plot_labels.append("All within %s" % field)
if not opts.suppress_all_between:
plot_data.append(get_all_grouped_distances(dist_matrix_header,
dist_matrix, mapping_header, mapping, field, within=False))
plot_labels.append("All between %s" % field)
if not opts.suppress_individual_within:
within_dists = get_grouped_distances(dist_matrix_header,
dist_matrix, mapping_header, mapping, field, within=True)
for grouping in within_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
if not opts.suppress_individual_between:
between_dists = get_grouped_distances(dist_matrix_header,
dist_matrix, mapping_header, mapping, field, within=False)
for grouping in between_dists:
plot_data.append(grouping[2])
plot_labels.append("%s vs. %s" % (grouping[0], grouping[1]))
# We now have our data and labels ready, so plot them!
assert (len(plot_data) == len(plot_labels)), "The number " +\
"of boxplot labels does not match the number of " +\
"boxplots."
if plot_data:
if opts.sort:
# Sort our plot data in order of increasing median.
sorted_data = []
for label, distribution in zip(plot_labels, plot_data):
sorted_data.append((label, distribution,
median(distribution)))
sorted_data.sort(key=itemgetter(2))
plot_labels = []
plot_data = []
for label, distribution, median_value in sorted_data:
plot_labels.append(label)
plot_data.append(distribution)
width = opts.width
height = opts.height
if width is None:
width = len(plot_data) * opts.box_width + 2
if width <= 0 or height <= 0:
option_parser.error("The specified width and height of the "
"image must be greater than zero.")
plot_figure = generate_box_plots(plot_data,
x_tick_labels=plot_labels, title="%s Distances" % field,
x_label="Grouping", y_label="Distance",
x_tick_labels_orientation='vertical', y_min=y_min,
y_max=y_max, whisker_length=opts.whisker_length,
box_width=opts.box_width, box_color=opts.box_color,
figure_width=width, figure_height=height)
output_plot_fp = join(opts.output_dir, "%s_Distances.%s"
% (field, opts.imagetype))
plot_figure.savefig(output_plot_fp, format=opts.imagetype,
transparent=opts.transparent)
else:
option_parser.error("You have chosen to suppress all plots. At "
"least one type of plot must be unsuppressed.")
if not opts.suppress_significance_tests:
sig_tests_f = open(join(
opts.output_dir, "%s_Stats.xls" % field), 'w')
sig_tests_results = all_pairs_t_test(plot_labels, plot_data,
tail_type=opts.tail_type,
num_permutations=opts.num_permutations)
sig_tests_f.write(sig_tests_results)
sig_tests_f.close()
if opts.save_raw_data:
# Write the raw plot data into a tab-delimited file.
assert(len(plot_labels) == len(plot_data))
raw_data_fp = join(opts.output_dir, "%s_Distances.xls"
% field)
raw_data_f = open(raw_data_fp, 'w')
for label, data in zip(plot_labels, plot_data):
raw_data_f.write(label.replace(" ", "_") + "\t")
raw_data_f.write("\t".join(map(str, data)))
raw_data_f.write("\n")
raw_data_f.close()