def main(args):
OptionParser.format_description = lambda self, formatter: self.description
OptionParser.format_epilog = lambda self, formatter: self.epilog
parser = OptionParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_option("", "--url", dest="url", default=API_URL, help="communities API url")
parser.add_option("", "--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_option("", "--output", dest="output", default='-', help="output: filename or stdout (-), default is stdout")
parser.add_option("", "--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_option("", "--cluster", dest="cluster", default='ward', help="cluster function, one of: ward, single, complete, mcquitty, median, centroid, default is ward")
parser.add_option("", "--distance", dest="distance", default='bray-curtis', help="distance function, one of: bray-curtis, euclidean, maximum, manhattan, canberra, minkowski, difference, default is bray-curtis")
parser.add_option("", "--name", dest="name", type="int", default=0, help="label columns by name, default is by id: 1=true, 0=false")
parser.add_option("", "--normalize", dest="normalize", type="int", default=0, help="normalize the input data, default is off: 1=true, 0=false")
# get inputs
(opts, args) = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
# parse inputs
rows = []
cols = []
data = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom, col_name=opts.name)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# retrieve data
raw = '0' if opts.normalize else '1'
post = {"raw": raw, "cluster": opts.cluster, "distance": opts.distance, "columns": cols, "rows": rows, "data": data}
hmap = obj_from_url(opts.url+'/compute/heatmap', data=json.dumps(post, separators=(',',':')))
# output data
if (not opts.output) or (opts.output == '-'):
out_hdl = sys.stdout
else:
out_hdl = open(opts.output, 'w')
out_hdl.write(json.dumps(hmap, separators=(', ',': '), indent=4)+"\n")
out_hdl.close()
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_argument("--user", dest="user", default=None, help="OAuth username")
parser.add_argument("--passwd", dest="passwd", default=None, help="OAuth password")
parser.add_argument("--token", dest="token", default=None, help="OAuth token")
parser.add_argument("--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_argument("--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--plot", dest="plot", default=None, help="filename for output plot")
parser.add_argument("--distance", dest="distance", default='bray-curtis', help="distance metric, one of: bray-curtis, euclidean, maximum, manhattan, canberra, minkowski, difference, default is bray-curtis")
parser.add_argument("--metadata", dest="metadata", default=None, help="metadata field to color by, only for 'biom' input")
parser.add_argument("--groups", dest="groups", default=None, help="list of groups in JSON or tabbed format - either as input string or filename")
parser.add_argument("--group_pos", dest="group_pos", type=int, default=1, help="position of group to use, default is 1 (first)")
parser.add_argument("--color_auto", dest="color_auto", type=int, default=0, help="auto-create colors based on like group names, default is use group name as color: 1=true, 0=false")
parser.add_argument("--rlib", dest="rlib", default=None, help="R lib path")
parser.add_argument("--height", dest="height", type=float, default=10, help="image height in inches, default is 6")
parser.add_argument("--width", dest="width", type=float, default=10, help="image width in inches, default is 6")
parser.add_argument("--dpi", dest="dpi", type=int, default=300, help="image DPI, default is 300")
parser.add_argument("--three", dest="three", type=int, default=0, help="create 3-D PCoA, default is 2-D: 1=true, 0=false")
parser.add_argument("--name", dest="name", type=int, default=0, help="label columns by name, default is by id: 1=true, 0=false")
parser.add_argument("--label", dest="label", type=int, default=0, help="label image rows, default is off: 1=true, 0=false")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if not opts.plot:
sys.stderr.write("ERROR: missing output filename\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
if not opts.rlib:
sys.stderr.write("ERROR: missing path to R libs\n")
return 1
if opts.metadata:
opts.color_auto = 1
for o in ['reference', 'color_auto', 'three', 'name', 'label']:
if getattr(opts, o) not in [0, 1]:
sys.stderr.write("ERROR: invalid value for '%s'\n"%o)
return 1
# get auth
token = get_auth_token(opts)
# parse inputs
tmp_in = 'tmp_'+random_str()+'.txt'
tmp_hdl = open(tmp_in, 'w')
mg_list = []
groups = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
indata = json.loads(indata)
mg_list = map(lambda x: x['id'], indata['columns'])
col_name = True if opts.name == 1 else False
biom_to_tab(indata, tmp_hdl, col_name=col_name)
if opts.metadata:
groups = metadata_from_biom(indata, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
tmp_hdl.write(indata)
mg_list = indata.split('\n')[0].strip().split('\t')
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
tmp_hdl.close()
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
return 1
for m in mg_list:
found_g = None
for g, mgs in gdata[opts.group_pos-1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
for m in mg_list:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos-1])
except:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# print groups to file for R input
tmp_group = None
if len(groups) == len(mg_list):
tmp_group = 'tmp_'+random_str()+'.txt'
hdl_group = open(tmp_group, 'w')
hdl_group.write("\tgroup\n")
for i, m in enumerate(mg_list):
hdl_group.write("%s\t%s\n"%(m, ''.join([x if ord(x) < 128 else '?' for x in groups[i]])))
hdl_group.close()
elif len(groups) > 0:
sys.stderr.write("Warning: Not all metagenomes in a group\n")
# build R cmd
three = 'c(1,2,3)' if opts.three == 1 else 'c(1,2)'
label = 'TRUE' if opts.label == 1 else 'FALSE'
table = '"%s"'%tmp_group if tmp_group else 'NA'
color = 'TRUE' if opts.color_auto == 1 else 'FALSE'
r_cmd = """source("%s/plot_mg_pcoa.r")
suppressMessages( plot_mg_pcoa(
table_in="%s",
image_out="%s",
plot_pcs=%s,
dist_metric="%s",
label_points=%s,
color_table=%s,
color_column=1,
auto_colors=%s,
image_height_in=%.1f,
image_width_in=%.1f,
image_res_dpi=%d
))"""%(opts.rlib, tmp_in, opts.plot, three, opts.distance, label, table, color, opts.height, opts.width, opts.dpi)
execute_r(r_cmd)
# cleanup
os.remove(tmp_in)
if tmp_group:
os.remove(tmp_group)
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
parser.add_argument("--url",
dest="url",
default=API_URL,
help="communities API url")
parser.add_argument("--rlib", dest="rlib", default=None, help="R lib path")
parser.add_argument("--input",
dest="input",
default='-',
help="input: filename or stdin (-), default is stdin")
parser.add_argument(
"--output",
dest="output",
default='-',
help="output: filename or stdout (-), default is stdout")
parser.add_argument(
"--outdir",
dest="outdir",
default=None,
help=
"ouput is placed in dir as filenmae.obj, fielname.type, only for 'biom' input"
)
parser.add_argument(
"--format",
dest="format",
default='biom',
help=
"input / output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom"
)
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid format\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
# parse inputs
biom = None
rows = []
cols = []
data = []
maxval = 0
tmp_in = 'tmp_' + random_str() + '.txt'
tmp_hdl = open(tmp_in, 'w')
try:
indata = sys.stdin.read() if opts.input == '-' else open(
opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
if opts.rlib:
maxval = biom_to_tab(biom, tmp_hdl)
else:
rows, cols, data = biom_to_matrix(biom)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
data = map(lambda x: map(float, x), data) # floatify it
if opts.rlib:
tmp_hdl.write(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
finally:
tmp_hdl.close()
# check values to see if already normalized, otherwise R fails badly
data = list(data)
if len(data) > 0:
maxval = max(map(max, data))
if maxval <= 1:
os.remove(tmp_in)
sys.stderr.write("ERROR: data is already normalized.\n")
return 1
# retrieve data
norm = None
if opts.rlib:
tmp_out = 'tmp_' + random_str() + '.txt'
r_cmd = """source("%s/preprocessing.r")
suppressMessages( MGRAST_preprocessing(
file_in="%s",
file_out="%s"
))""" % (opts.rlib, tmp_in, tmp_out)
execute_r(r_cmd)
nrows, ncols, ndata = tab_to_matrix(open(tmp_out, 'r').read())
num_data = map(lambda x: map(float, x), ndata)
norm = {"columns": ncols, "rows": nrows, "data": num_data}
os.remove(tmp_out)
else:
post = {"columns": cols, "rows": rows, "data": data}
norm = obj_from_url(opts.url + '/compute/normalize',
data=json.dumps(post, separators=(',', ':')))
# output data
os.remove(tmp_in)
if (not opts.output) or (opts.output == '-'):
out_hdl = sys.stdout
else:
out_hdl = open(opts.output, 'w')
if biom and (opts.format == 'biom'):
# may have rows removed
new_rows = []
for r in biom['rows']:
if r['id'] in norm['rows']:
new_rows.append(r)
biom['rows'] = new_rows
biom['data'] = norm['data']
biom['shape'][0] = len(biom['rows'])
biom['id'] = biom['id'] + '_normalized'
biom['matrix_type'] = 'dense'
biom['matrix_element_type'] = 'float'
matrix_type = None
if biom['type'].startswith('Taxon'):
matrix_type = "Communities.TaxonomicMatrix"
elif biom['type'].startswith('Function'):
matrix_type = "Communities.FunctionalMatrix"
if opts.outdir and matrix_type:
if not os.path.isdir(opts.outdir):
os.mkdir(opts.outdir)
ohdl = open(os.path.join(opts.outdir, opts.output + '.obj'), 'w')
thdl = open(os.path.join(opts.outdir, opts.output + '.type'), 'w')
ohdl.write(json.dumps(biom) + "\n")
thdl.write(matrix_type)
ohdl.close()
thdl.close()
else:
out_hdl.write(json.dumps(biom) + "\n")
else:
out_hdl.write("\t%s\n" % "\t".join(norm['columns']))
for i, d in enumerate(norm['data']):
out_hdl.write("%s\t%s\n" %
(norm['rows'][i], "\t".join(map(str, d))))
out_hdl.close()
if os.stat(opts.output).st_size == 0:
os.remove(opts.output)
return 0
def main(args):
OptionParser.format_description = lambda self, formatter: self.description
OptionParser.format_epilog = lambda self, formatter: self.epilog
parser = OptionParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_option("", "--url", dest="url", default=API_URL, help="communities API url")
parser.add_option("", "--rlib", dest="rlib", default=None, help="R lib path")
parser.add_option("", "--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_option("", "--output", dest="output", default='-', help="output: filename or stdout (-), default is stdout")
parser.add_option("", "--outdir", dest="outdir", default=None, help="ouput is placed in dir as filenmae.obj, fielname.type, only for 'biom' input")
parser.add_option("", "--format", dest="format", default='biom', help="input / output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
# get inputs
(opts, args) = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid format\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
# parse inputs
biom = None
rows = []
cols = []
data = []
maxval = 0
tmp_in = 'tmp_'+random_str()+'.txt'
tmp_hdl = open(tmp_in, 'w')
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
if opts.rlib:
maxval = biom_to_tab(biom, tmp_hdl)
else:
rows, cols, data = biom_to_matrix(biom)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
data = map(lambda x: map(float, x), data) # floatify it
if opts.rlib:
tmp_hdl.write(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
finally:
tmp_hdl.close()
# check values to see if already normalized, otherwise R fails badly
if len(data) > 0:
maxval = max( map(max, data) )
if maxval <= 1:
os.remove(tmp_in)
sys.stderr.write("ERROR: data is already normalized.\n")
return 1
# retrieve data
norm = None
if opts.rlib:
tmp_out = 'tmp_'+random_str()+'.txt'
r_cmd = """source("%s/preprocessing.r")
suppressMessages( MGRAST_preprocessing(
file_in="%s",
file_out="%s"
))"""%(opts.rlib, tmp_in, tmp_out)
execute_r(r_cmd)
nrows, ncols, ndata = tab_to_matrix(open(tmp_out, 'r').read())
num_data = map(lambda x: map(float, x), ndata)
norm = {"columns": ncols, "rows": nrows, "data": num_data}
os.remove(tmp_out)
else:
post = {"columns": cols, "rows": rows, "data": data}
norm = obj_from_url(opts.url+'/compute/normalize', data=json.dumps(post, separators=(',',':')))
# output data
os.remove(tmp_in)
if (not opts.output) or (opts.output == '-'):
out_hdl = sys.stdout
else:
out_hdl = open(opts.output, 'w')
if biom and (opts.format == 'biom'):
# may have rows removed
new_rows = []
for r in biom['rows']:
if r['id'] in norm['rows']:
new_rows.append(r)
biom['rows'] = new_rows
biom['data'] = norm['data']
biom['shape'][0] = len(biom['rows'])
biom['id'] = biom['id']+'_normalized'
biom['matrix_type'] = 'dense'
biom['matrix_element_type'] = 'float'
matrix_type = None
if biom['type'].startswith('Taxon'):
matrix_type = "Communities.TaxonomicMatrix"
elif biom['type'].startswith('Function'):
matrix_type = "Communities.FunctionalMatrix"
if opts.outdir and matrix_type:
if not os.path.isdir(opts.outdir):
os.mkdir(opts.outdir)
ohdl = open(os.path.join(opts.outdir, opts.output+'.obj'), 'w')
thdl = open(os.path.join(opts.outdir, opts.output+'.type'), 'w')
ohdl.write(json.dumps(biom)+"\n")
thdl.write(matrix_type)
ohdl.close()
thdl.close()
else:
out_hdl.write(json.dumps(biom)+"\n")
else:
out_hdl.write( "\t%s\n" %"\t".join(norm['columns']) )
for i, d in enumerate(norm['data']):
out_hdl.write( "%s\t%s\n" %(norm['rows'][i], "\t".join(map(str, d))) )
out_hdl.close()
if os.stat(opts.output).st_size == 0:
os.remove(opts.output);
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
parser.add_argument("--url",
dest="url",
default=API_URL,
help="communities API url")
parser.add_argument("--input",
dest="input",
default='-',
help="input: filename or stdin (-), default is stdin")
parser.add_argument(
"--output",
dest="output",
default='-',
help="output: filename or stdout (-), default is stdout")
parser.add_argument(
"--format",
dest="format",
default='biom',
help=
"input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom"
)
parser.add_argument(
"--cluster",
dest="cluster",
default='ward',
help=
"cluster function, one of: ward, single, complete, mcquitty, median, centroid, default is ward"
)
parser.add_argument(
"--distance",
dest="distance",
default='bray-curtis',
help=
"distance function, one of: bray-curtis, euclidean, maximum, manhattan, canberra, minkowski, difference, default is bray-curtis"
)
parser.add_argument(
"--name",
dest="name",
type=int,
default=0,
help="label columns by name, default is by id: 1=true, 0=false")
parser.add_argument(
"--normalize",
dest="normalize",
type=int,
default=0,
help="normalize the input data, default is off: 1=true, 0=false")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
# parse inputs
rows = []
cols = []
data = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(
opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom, col_name=opts.name)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# retrieve data
raw = '0' if opts.normalize else '1'
post = {
"raw": raw,
"cluster": opts.cluster,
"distance": opts.distance,
"columns": cols,
"rows": rows,
"data": data
}
hmap = obj_from_url(opts.url + '/compute/heatmap',
data=json.dumps(post, separators=(',', ':')))
# output data
if (not opts.output) or (opts.output == '-'):
out_hdl = sys.stdout
else:
out_hdl = open(opts.output, 'w')
out_hdl.write(json.dumps(hmap, separators=(', ', ': '), indent=4) + "\n")
out_hdl.close()
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
parser.add_argument("--url",
dest="url",
default=API_URL,
help="communities API url")
parser.add_argument("--input",
dest="input",
default='-',
help="input: filename or stdin (-), default is stdin")
parser.add_argument(
"--output",
dest="output",
default='-',
help="output: filename or stdout (-), default is stdout")
parser.add_argument(
"--format",
dest="format",
default='biom',
help=
"input / output format: 'text' for tabbed table, 'biom' for BIOM / json format, default is biom"
)
parser.add_argument(
"--metadata",
dest="metadata",
default=None,
help="metadata field to group by, only for 'biom' input")
parser.add_argument(
"--distance",
dest="distance",
default='bray-curtis',
help=
"distance metric, one of: bray-curtis, euclidean, maximum, manhattan, canberra, minkowski, difference, default is bray-curtis"
)
parser.add_argument(
"--name",
dest="name",
type=int,
default=0,
help="label columns by name, default is by id: 1=true, 0=false")
parser.add_argument(
"--normalize",
dest="normalize",
type=int,
default=0,
help="normalize the input data, default is off: 1=true, 0=false")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
# parse inputs
rows = []
cols = []
data = []
groups = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(
opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
col_name = True if opts.name == 1 else False
rows, cols, data = biom_to_matrix(biom, col_name=col_name)
if opts.metadata:
groups = metadata_from_biom(biom, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# get group map
gmap = {}
for i, g in enumerate(groups):
if g != 'null':
gmap[cols[i]] = g
# retrieve data
raw = '0' if opts.normalize == 1 else '1'
post = {
"raw": raw,
"distance": opts.distance,
"columns": cols,
"rows": rows,
"data": data
}
pcoa = obj_from_url(opts.url + '/compute/pcoa',
data=json.dumps(post, separators=(',', ':')))
# output data
if (not opts.output) or (opts.output == '-'):
out_hdl = sys.stdout
else:
out_hdl = open(opts.output, 'w')
if opts.format == 'biom':
for i in range(len(pcoa['data'])):
pcoa['data'][i]['group'] = gmap[
pcoa['data'][i]['id']] if pcoa['data'][i]['id'] in gmap else ""
out_hdl.write(json.dumps(pcoa) + "\n")
else:
out_hdl.write("ID\tGroup\tPC1\tPC2\tPC3\tPC4\n")
for d in pcoa['data']:
out_hdl.write("%s\t%s\t%s\n" %
(d['id'], gmap[d['id']] if
(d['id'] in gmap) else "", "\t".join(
map(str, d['pco'][0:4]))))
out_hdl.close()
return 0
def main(args):
OptionParser.format_description = lambda self, formatter: self.description
OptionParser.format_epilog = lambda self, formatter: self.epilog
parser = OptionParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_option("", "--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_option("", "--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_option("", "--output", dest="output", default='biom', help="output format: 'full' for tabbed abundances and stats, 'minimum' for tabbed stats only, 'biom' for BIOM format, default is biom")
parser.add_option("", "--metadata", dest="metadata", default=None, help="metadata field to correlate, only for 'biom' input")
parser.add_option("", "--groups", dest="groups", default=None, help="list of groups in JSON or tabbed format - either as input string or filename")
parser.add_option("", "--group_pos", dest="group_pos", type="int", default=1, help="position of group to use, default is 1 (first)")
parser.add_option("", "--cutoff", dest="cutoff", default=None, help="only show p-value less than this, default show all")
parser.add_option("", "--fdr", dest="fdr", action="store_true", default=False, help="output FDR for computed p-values, default is off")
# get inputs
(opts, args) = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['full', 'minimum', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
# parse inputs
biom = None
rows = []
cols = []
data = []
groups = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom)
if opts.metadata:
groups = metadata_from_biom(biom, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write("ERROR: position (%d) of metadata is out of bounds\n"%opts.group_pos)
return 1
for m in cols:
found_g = None
for g, mgs in gdata[opts.group_pos-1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s missing metadata\n"%m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write("ERROR: position (%d) of metadata is out of bounds\n"%opts.group_pos)
for m in cols:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos-1])
except:
sys.stderr.write("ERROR: metagenome %s missing metadata\n"%m)
return 1
# validate metadata
if len(groups) != len(cols):
sys.stderr.write("ERROR: Not all metagenomes have metadata\n")
return 1
try:
groups = map(lambda x: float(x), groups)
except:
sys.stderr.write("ERROR: Metadata is not numeric\n")
return 1
# check correlation
results = []
pvalues = []
for i, a in enumerate(rows): # annotations
l_meta = []
l_anno = []
for j, m in enumerate(cols): # metagenomes
l_meta.append(groups[j])
l_anno.append(float(data[i][j]))
gradient, intercept, r_value, p_value, std_err = stats.linregress(l_meta, l_anno)
if biom and (opts.output == 'biom'):
results.append([('r-value', r_value), ('p-value', p_value)])
pvalues.append(p_value)
else:
if opts.output == 'full':
l_result = [a]+[float(x) for x in data[i]]+[r_value, p_value]
elif opts.output == 'minimum':
l_result = [a, r_value, p_value]
if (not opts.cutoff) or (opts.cutoff and (p_value < opts.cutoff)):
results.append(l_result)
pvalues.append(p_value)
# calculate fdr
if opts.fdr and pvalues:
fdr_values = calculate_fdr(pvalues)
for i, x in enumerate(fdr_values):
results[i].append(('fdr', x) if biom and (opts.output == 'biom') else x)
# output
if biom and (opts.output == 'biom'):
# add stats to row data, same order
new_rows = []
for i, robj in enumerate(biom['rows']):
if not robj['metadata']:
robj['metadata'] = {'correlate': results[i]}
else:
robj['metadata']['correlate'] = results[i]
new_rows.append(robj)
# update biom
biom['id'] = biom['id']+'_corr'
biom['rows'] = new_rows
safe_print(json.dumps(biom)+'\n')
else:
header = ['r-value', 'p-value']
if opts.output == 'full':
header = header
if opts.fdr:
header.append('fdr')
safe_print("\t%s\n"%"\t".join(header))
for r in results:
safe_print(r[0])
for x in r[1:]:
if int(x) == float(x):
safe_print("\t%d"%int(x))
else:
safe_print("\t%.5f"%float(x))
safe_print("\n")
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_argument("--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_argument("--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--output", dest="output", default='biom', help="output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--order", dest="order", type=int, default=None, help="column number to order output by (0 for last column), default is no ordering")
parser.add_argument("--direction", dest="direction", default="desc", help="direction of order. 'asc' for ascending order, 'desc' for descending order, default is desc")
parser.add_argument("--cols", dest="cols", type=int, default=None, help="number of columns from the left to return from input table, default is all")
parser.add_argument("--rows", dest="rows", type=int, default=None, help="number of rows from the top to return from input table, default is all")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
if opts.direction not in ['asc', 'desc']:
sys.stderr.write("ERROR: invalid order direction\n")
return 1
# parse inputs
biom = None
rows = []
cols = []
data = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom, sig_stats=True)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# first we sort
if opts.order is not None:
rev_order = True if opts.direction == 'desc' else False
order_col = opts.order
if order_col > len(cols):
sys.stderr.write("ERROR: --order value is greater than number of columns in table\n")
order_col = order_col - 1
rd_merged = zip(rows, data)
rd_sorted = sorted(rd_merged, key=lambda x: x[1][order_col], reverse=rev_order)
rows, data = zip(*rd_sorted)
rows, data = list(rows), list(data)
# subselect rows
if opts.rows is not None:
subrow = opts.rows
rows = rows[:subrow]
data = data[:subrow]
if opts.cols is not None:
subcol = opts.cols
cols = cols[:subcol]
data = sub_matrix(data, subcol)
# output data
if biom and (opts.output == 'biom'):
# get list of new rows and columns with old indexes
br_ids = [r['id'] for r in biom['rows']]
bc_ids = [c['id'] for c in biom['columns']]
rindex = []
cindex = []
for r in rows:
try:
rindex.append(br_ids.index(r))
except:
pass
for c in cols:
try:
cindex.append(bc_ids.index(c))
except:
pass
# update biom
biom['id'] = biom['id']+'_altered'
biom['data'] = sub_matrix(data, biom['shape'][1])
biom['rows'] = [biom['rows'][r] for r in rindex]
biom['columns'] = [biom['columns'][c] for c in cindex]
biom['shape'] = [len(biom['rows']), len(biom['columns'])]
biom['matrix_type'] = 'dense'
safe_print(json.dumps(biom)+'\n')
else:
safe_print("\t%s\n" %"\t".join(cols))
for i, d in enumerate(data):
safe_print("%s\t%s\n" %(rows[i], "\t".join(map(str, d))))
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_argument("--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_argument("--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--output", dest="output", default='biom', help="output format: 'full' for tabbed abundances and stats, 'minimum' for tabbed stats only, 'biom' for BIOM format, default is biom")
parser.add_argument("--metadata", dest="metadata", default=None, help="metadata field to correlate, only for 'biom' input")
parser.add_argument("--groups", dest="groups", default=None, help="list of groups in JSON or tabbed format - either as input string or filename")
parser.add_argument("--group_pos", dest="group_pos", type=int, default=1, help="position of group to use, default is 1 (first)")
parser.add_argument("--cutoff", dest="cutoff", default=None, help="only show p-value less than this, default show all")
parser.add_argument("--fdr", dest="fdr", action="store_true", default=False, help="output FDR for computed p-values, default is off")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['full', 'minimum', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
# parse inputs
biom = None
rows = []
cols = []
data = []
groups = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom)
if opts.metadata:
groups = metadata_from_biom(biom, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write("ERROR: position (%d) of metadata is out of bounds\n"%opts.group_pos)
return 1
for m in cols:
found_g = None
for g, mgs in gdata[opts.group_pos-1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s missing metadata\n"%m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write("ERROR: position (%d) of metadata is out of bounds\n"%opts.group_pos)
for m in cols:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos-1])
except:
sys.stderr.write("ERROR: metagenome %s missing metadata\n"%m)
return 1
# validate metadata
if len(groups) != len(cols):
sys.stderr.write("ERROR: Not all metagenomes have metadata\n")
return 1
try:
groups = map(lambda x: float(x), groups)
except:
sys.stderr.write("ERROR: Metadata is not numeric\n")
return 1
# check correlation
results = []
pvalues = []
for i, a in enumerate(rows): # annotations
l_meta = []
l_anno = []
for j, m in enumerate(cols): # metagenomes
l_meta.append(groups[j])
l_anno.append(float(data[i][j]))
gradient, intercept, r_value, p_value, std_err = stats.linregress(l_meta, l_anno)
if biom and (opts.output == 'biom'):
results.append([('r-value', r_value), ('p-value', p_value)])
pvalues.append(p_value)
else:
if opts.output == 'full':
l_result = [a]+[float(x) for x in data[i]]+[r_value, p_value]
elif opts.output == 'minimum':
l_result = [a, r_value, p_value]
if (not opts.cutoff) or (opts.cutoff and (p_value < opts.cutoff)):
results.append(l_result)
pvalues.append(p_value)
# calculate fdr
if opts.fdr and pvalues:
fdr_values = calculate_fdr(pvalues)
for i, x in enumerate(fdr_values):
results[i].append(('fdr', x) if biom and (opts.output == 'biom') else x)
# output
if biom and (opts.output == 'biom'):
# add stats to row data, same order
new_rows = []
for i, robj in enumerate(biom['rows']):
if not robj['metadata']:
robj['metadata'] = {'correlate': results[i]}
else:
robj['metadata']['correlate'] = results[i]
new_rows.append(robj)
# update biom
biom['id'] = biom['id']+'_corr'
biom['rows'] = new_rows
safe_print(json.dumps(biom)+'\n')
else:
header = ['r-value', 'p-value']
if opts.output == 'full':
header = header
if opts.fdr:
header.append('fdr')
safe_print("\t%s\n"%"\t".join(header))
for r in results:
safe_print(r[0])
for x in r[1:]:
if int(x) == float(x):
safe_print("\t%d"%int(x))
else:
safe_print("\t%.5f"%float(x))
safe_print("\n")
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
parser.add_argument("--user",
dest="user",
default=None,
help="OAuth username")
parser.add_argument("--passwd",
dest="passwd",
default=None,
help="OAuth password")
parser.add_argument("--token",
dest="token",
default=None,
help="OAuth token")
parser.add_argument("--input",
dest="input",
default='-',
help="input: filename or stdin (-), default is stdin")
parser.add_argument(
"--format",
dest="format",
default='biom',
help=
"input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom"
)
parser.add_argument("--plot",
dest="plot",
default=None,
help="filename for output plot")
parser.add_argument(
"--distance",
dest="distance",
default='bray-curtis',
help=
"distance metric, one of: bray-curtis, euclidean, maximum, manhattan, canberra, minkowski, difference, default is bray-curtis"
)
parser.add_argument(
"--metadata",
dest="metadata",
default=None,
help="metadata field to color by, only for 'biom' input")
parser.add_argument(
"--groups",
dest="groups",
default=None,
help=
"list of groups in JSON or tabbed format - either as input string or filename"
)
parser.add_argument("--group_pos",
dest="group_pos",
type=int,
default=1,
help="position of group to use, default is 1 (first)")
parser.add_argument(
"--color_auto",
dest="color_auto",
type=int,
default=0,
help=
"auto-create colors based on like group names, default is use group name as color: 1=true, 0=false"
)
parser.add_argument("--rlib", dest="rlib", default=None, help="R lib path")
parser.add_argument("--height",
dest="height",
type=float,
default=10,
help="image height in inches, default is 6")
parser.add_argument("--width",
dest="width",
type=float,
default=10,
help="image width in inches, default is 6")
parser.add_argument("--dpi",
dest="dpi",
type=int,
default=300,
help="image DPI, default is 300")
parser.add_argument(
"--three",
dest="three",
type=int,
default=0,
help="create 3-D PCoA, default is 2-D: 1=true, 0=false")
parser.add_argument(
"--name",
dest="name",
type=int,
default=0,
help="label columns by name, default is by id: 1=true, 0=false")
parser.add_argument(
"--label",
dest="label",
type=int,
default=0,
help="label image rows, default is off: 1=true, 0=false")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if not opts.plot:
sys.stderr.write("ERROR: missing output filename\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
if not opts.rlib:
sys.stderr.write("ERROR: missing path to R libs\n")
return 1
if opts.metadata:
opts.color_auto = 1
for o in ['reference', 'color_auto', 'three', 'name', 'label']:
if getattr(opts, o) not in [0, 1]:
sys.stderr.write("ERROR: invalid value for '%s'\n" % o)
return 1
# get auth
token = get_auth_token(opts)
# parse inputs
tmp_in = 'tmp_' + random_str() + '.txt'
tmp_hdl = open(tmp_in, 'w')
mg_list = []
groups = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(
opts.input, 'r').read()
if opts.format == 'biom':
try:
indata = json.loads(indata)
mg_list = map(lambda x: x['id'], indata['columns'])
col_name = True if opts.name == 1 else False
biom_to_tab(indata, tmp_hdl, col_name=col_name)
if opts.metadata:
groups = metadata_from_biom(indata, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
tmp_hdl.write(indata)
mg_list = indata.split('\n')[0].strip().split('\t')
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
tmp_hdl.close()
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(
opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write(
"ERROR: position (%d) of group is out of bounds\n" %
opts.group_pos)
return 1
for m in mg_list:
found_g = None
for g, mgs in gdata[opts.group_pos - 1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s not in a group\n" %
m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(
opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write(
"ERROR: position (%d) of group is out of bounds\n" %
opts.group_pos)
for m in mg_list:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos - 1])
except:
sys.stderr.write("ERROR: metagenome %s not in a group\n" %
m)
return 1
# print groups to file for R input
tmp_group = None
if len(groups) == len(mg_list):
tmp_group = 'tmp_' + random_str() + '.txt'
hdl_group = open(tmp_group, 'w')
hdl_group.write("\tgroup\n")
for i, m in enumerate(mg_list):
hdl_group.write(
"%s\t%s\n" %
(m, ''.join([x if ord(x) < 128 else '?' for x in groups[i]])))
hdl_group.close()
elif len(groups) > 0:
sys.stderr.write("Warning: Not all metagenomes in a group\n")
# build R cmd
three = 'c(1,2,3)' if opts.three == 1 else 'c(1,2)'
label = 'TRUE' if opts.label == 1 else 'FALSE'
table = '"%s"' % tmp_group if tmp_group else 'NA'
color = 'TRUE' if opts.color_auto == 1 else 'FALSE'
r_cmd = """source("%s/plot_mg_pcoa.r")
suppressMessages( plot_mg_pcoa(
table_in="%s",
image_out="%s",
plot_pcs=%s,
dist_metric="%s",
label_points=%s,
color_table=%s,
color_column=1,
auto_colors=%s,
image_height_in=%.1f,
image_width_in=%.1f,
image_res_dpi=%d
))""" % (opts.rlib, tmp_in, opts.plot, three, opts.distance, label, table,
color, opts.height, opts.width, opts.dpi)
execute_r(r_cmd)
# cleanup
os.remove(tmp_in)
if tmp_group:
os.remove(tmp_group)
return 0
def main(args):
OptionParser.format_description = lambda self, formatter: self.description
OptionParser.format_epilog = lambda self, formatter: self.epilog
parser = OptionParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_option("", "--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_option("", "--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_option("", "--output", dest="output", default='biom', help="output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_option("", "--plot", dest="plot", default=None, help="filename for output plot, optional")
parser.add_option("", "--stat_test", dest="stat_test", default='Kruskal-Wallis', help="supported statistical tests, one of: Kruskal-Wallis, t-test-paired, Wilcoxon-paired, t-test-unpaired, Mann-Whitney-unpaired-Wilcoxon, ANOVA-one-way, default is Kruskal-Wallis")
parser.add_option("", "--metadata", dest="metadata", default=None, help="metadata field to group by, only for 'biom' input")
parser.add_option("", "--groups", dest="groups", default=None, help="list of groups in JSON or tabbed format - either as input string or filename")
parser.add_option("", "--group_pos", dest="group_pos", type="int", default=1, help="position of group to use, default is 1 (first)")
parser.add_option("", "--rlib", dest="rlib", default=None, help="R lib path")
parser.add_option("", "--order", dest="order", default=None, help="column number to order output by, default is last column")
parser.add_option("", "--direction", dest="direction", default="desc", help="direction of order. 'asc' for ascending order, 'desc' for descending order, default is desc")
parser.add_option("", "--height", dest="height", type="float", default=6, help="image height in inches, default is 6")
parser.add_option("", "--width", dest="width", type="float", default=6, help="image width in inches, default is 6")
parser.add_option("", "--dpi", dest="dpi", type="int", default=300, help="image DPI, default is 300")
# get inputs
(opts, args) = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
if not opts.rlib:
sys.stderr.write("ERROR: missing path to R libs\n")
return 1
if opts.direction not in ['asc', 'desc']:
sys.stderr.write("ERROR: invalid order direction\n")
return 1
# parse inputs
tmp_in = 'tmp_'+random_str()+'.txt'
tmp_out = 'tmp_'+random_str()+'.txt'
tmp_hdl = open(tmp_in, 'w')
mg_list = []
groups = []
biom = None
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
mg_list = map(lambda x: x['id'], biom['columns'])
biom_to_tab(biom, tmp_hdl)
if opts.metadata:
groups = metadata_from_biom(biom, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
tmp_hdl.write(indata)
mg_list = indata.split('\n')[0].strip().split('\t')
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
tmp_hdl.close()
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
return 1
for m in mg_list:
found_g = None
for g, mgs in gdata[opts.group_pos-1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
for m in mg_list:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos-1])
except:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# validate groups
if len(groups) != len(mg_list):
sys.stderr.write("ERROR: Not all metagenomes in a group\n")
return 1
# build R cmd
fig_out = '"%s"'%opts.plot if opts.plot else 'NULL'
order_by = 'NULL' if opts.order is None else int(opts.order)
order_desc = 'TRUE' if opts.direction == 'desc' else 'FALSE'
group_str = 'c('+','.join(map(lambda x: '"%s"'%x, groups))+')'
r_cmd = """source("%s/group_stats_plot.r")
suppressMessages( group_stats_plot(
file_in="%s",
file_out="%s",
figure_out=%s,
figure_width_in=%.1f,
figure_height_in=%.1f,
figure_res_dpi=%d,
stat_test="%s",
order_by=%s,
order_decreasing=%s,
my_grouping=%s
))"""%(opts.rlib, tmp_in, tmp_out, fig_out, opts.height, opts.width, opts.dpi, opts.stat_test, order_by, order_desc, group_str)
execute_r(r_cmd)
# output results
results = open(tmp_out, 'r').read()
os.remove(tmp_in)
os.remove(tmp_out)
if biom and (opts.output == 'biom'):
cnum = biom['shape'][1]
rids = [r['id'] for r in biom['rows']]
rrows, rcols, rdata = tab_to_matrix(results)
if (len(rrows) != biom['shape'][0]) or (len(rcols[:cnum]) != biom['shape'][1]):
sys.stderr.write("ERROR: significance test returned invalid results\n")
return 1
# add stats to row data, re-order
new_rows = []
new_data = []
for i, row in enumerate(rdata):
# get row
rindex = rids.index(rrows[i])
robj = biom['rows'][rindex]
if not robj['metadata']:
robj['metadata'] = {'significance': []}
else:
robj['metadata']['significance'] = []
# add stats
for j, stat in enumerate(row[cnum:]):
try:
stat = float(stat)
except:
stat = None
robj['metadata']['significance'].append((rcols[cnum:][j], stat))
new_rows.append(robj)
new_data.append(row[:cnum])
# update biom
biom['id'] = biom['id']+'_sig'
biom['rows'] = new_rows
biom['data'] = new_data
biom['matrix_type'] = 'dense'
for i, g in enumerate(groups):
biom['columns'][i]['group'] = g
safe_print(json.dumps(biom)+'\n')
else:
safe_print(results)
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
parser.add_argument("--input", dest="input", default='-', help="input: filename or stdin (-), default is stdin")
parser.add_argument("--format", dest="format", default='biom', help="input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--output", dest="output", default='biom', help="output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom")
parser.add_argument("--plot", dest="plot", default=None, help="filename for output plot, optional")
parser.add_argument("--stat_test", dest="stat_test", default='Kruskal-Wallis', help="supported statistical tests, one of: Kruskal-Wallis, t-test-paired, Wilcoxon-paired, t-test-unpaired, Mann-Whitney-unpaired-Wilcoxon, ANOVA-one-way, default is Kruskal-Wallis")
parser.add_argument("--metadata", dest="metadata", default=None, help="metadata field to group by, only for 'biom' input")
parser.add_argument("--groups", dest="groups", default=None, help="list of groups in JSON or tabbed format - either as input string or filename")
parser.add_argument("--group_pos", dest="group_pos", type=int, default=1, help="position of group to use, default is 1 (first)")
parser.add_argument("--rlib", dest="rlib", default=None, help="R lib path")
parser.add_argument("--order", dest="order", default=None, help="column number to order output by, default is last column")
parser.add_argument("--direction", dest="direction", default="desc", help="direction of order. 'asc' for ascending order, 'desc' for descending order, default is desc")
parser.add_argument("--height", dest="height", type=float, default=6, help="image height in inches, default is 6")
parser.add_argument("--width", dest="width", type=float, default=6, help="image width in inches, default is 6")
parser.add_argument("--dpi", dest="dpi", type=int, default=300, help="image DPI, default is 300")
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
if (not opts.rlib) and ('KB_PERL_PATH' in os.environ):
opts.rlib = os.environ['KB_PERL_PATH']
if not opts.rlib:
sys.stderr.write("ERROR: missing path to R libs\n")
return 1
if opts.direction not in ['asc', 'desc']:
sys.stderr.write("ERROR: invalid order direction\n")
return 1
# parse inputs
tmp_in = 'tmp_'+random_str()+'.txt'
tmp_out = 'tmp_'+random_str()+'.txt'
tmp_hdl = open(tmp_in, 'w')
mg_list = []
groups = []
biom = None
try:
indata = sys.stdin.read() if opts.input == '-' else open(opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
mg_list = map(lambda x: x['id'], biom['columns'])
biom_to_tab(biom, tmp_hdl)
if opts.metadata:
groups = metadata_from_biom(biom, opts.metadata)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
tmp_hdl.write(indata)
mg_list = indata.split('\n')[0].strip().split('\t')
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
tmp_hdl.close()
# get groups if not in BIOM metadata and option used
if (len(groups) == 0) and opts.groups:
# is it json ?
## example of 2 group sets in json format
## [ {"group1": ["mg_id_1", "mg_id_2"], "group2": ["mg_id_3", "mg_id_4", "mg_id_5"]},
## {"group1": ["mg_id_1", "mg_id_2", "mg_id_3"], "group2": ["mg_id_4", "mg_id_5"]} ]
try:
gdata = json.load(open(opts.groups, 'r')) if os.path.isfile(opts.groups) else json.loads(opts.groups)
if opts.group_pos > len(gdata):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
return 1
for m in mg_list:
found_g = None
for g, mgs in gdata[opts.group_pos-1].items():
if m in mgs:
found_g = g
break
if found_g:
groups.append(found_g)
else:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# no - its tabbed
except:
gtext = open(opts.groups, 'r').read() if os.path.isfile(opts.groups) else opts.groups
grows, gcols, gdata = tab_to_matrix(gtext)
if opts.group_pos > len(gdata[0]):
sys.stderr.write("ERROR: position (%d) of group is out of bounds\n"%opts.group_pos)
for m in mg_list:
try:
midx = gcols.index(m)
groups.append(gdata[midx][opts.group_pos-1])
except:
sys.stderr.write("ERROR: metagenome %s not in a group\n"%m)
return 1
# validate groups
if len(groups) != len(mg_list):
sys.stderr.write("ERROR: Not all metagenomes in a group\n")
return 1
# build R cmd
fig_out = '"%s"'%opts.plot if opts.plot else 'NULL'
order_by = 'NULL' if opts.order is None else int(opts.order)
order_desc = 'TRUE' if opts.direction == 'desc' else 'FALSE'
group_str = 'c('+','.join(map(lambda x: '"%s"'%x, groups))+')'
r_cmd = """source("%s/group_stats_plot.r")
suppressMessages( group_stats_plot(
file_in="%s",
file_out="%s",
figure_out=%s,
figure_width_in=%.1f,
figure_height_in=%.1f,
figure_res_dpi=%d,
stat_test="%s",
order_by=%s,
order_decreasing=%s,
my_grouping=%s
))"""%(opts.rlib, tmp_in, tmp_out, fig_out, opts.height, opts.width, opts.dpi, opts.stat_test, order_by, order_desc, group_str)
execute_r(r_cmd)
# output results
results = open(tmp_out, 'r').read()
os.remove(tmp_in)
os.remove(tmp_out)
if biom and (opts.output == 'biom'):
cnum = biom['shape'][1]
rids = [r['id'] for r in biom['rows']]
rrows, rcols, rdata = tab_to_matrix(results)
if (len(rrows) != biom['shape'][0]) or (len(rcols[:cnum]) != biom['shape'][1]):
sys.stderr.write("ERROR: significance test returned invalid results\n")
return 1
# add stats to row data, re-order
new_rows = []
new_data = []
for i, row in enumerate(rdata):
# get row
rindex = rids.index(rrows[i])
robj = biom['rows'][rindex]
if not robj['metadata']:
robj['metadata'] = {'significance': []}
else:
robj['metadata']['significance'] = []
# add stats
for j, stat in enumerate(row[cnum:]):
try:
stat = float(stat)
except:
stat = None
robj['metadata']['significance'].append((rcols[cnum:][j], stat))
new_rows.append(robj)
new_data.append(row[:cnum])
# update biom
biom['id'] = biom['id']+'_sig'
biom['rows'] = new_rows
biom['data'] = new_data
biom['matrix_type'] = 'dense'
for i, g in enumerate(groups):
biom['columns'][i]['group'] = g
safe_print(json.dumps(biom)+'\n')
else:
safe_print(results)
return 0
def main(args):
ArgumentParser.format_description = lambda self, formatter: self.description
ArgumentParser.format_epilog = lambda self, formatter: self.epilog
parser = ArgumentParser(usage='',
description=prehelp % VERSION,
epilog=posthelp % AUTH_LIST)
parser.add_argument("--input",
dest="input",
default='-',
help="input: filename or stdin (-), default is stdin")
parser.add_argument(
"--format",
dest="format",
default='biom',
help=
"input format: 'text' for tabbed table, 'biom' for BIOM format, default is biom"
)
parser.add_argument(
"--output",
dest="output",
default='biom',
help=
"output format: 'text' for tabbed table, 'biom' for BIOM format, default is biom"
)
parser.add_argument(
"--order",
dest="order",
type=int,
default=None,
help=
"column number to order output by (0 for last column), default is no ordering"
)
parser.add_argument(
"--direction",
dest="direction",
default="desc",
help=
"direction of order. 'asc' for ascending order, 'desc' for descending order, default is desc"
)
parser.add_argument(
"--cols",
dest="cols",
type=int,
default=None,
help=
"number of columns from the left to return from input table, default is all"
)
parser.add_argument(
"--rows",
dest="rows",
type=int,
default=None,
help=
"number of rows from the top to return from input table, default is all"
)
# get inputs
opts = parser.parse_args()
if (opts.input != '-') and (not os.path.isfile(opts.input)):
sys.stderr.write("ERROR: input data missing\n")
return 1
if opts.format not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid input format\n")
return 1
if opts.output not in ['text', 'biom']:
sys.stderr.write("ERROR: invalid output format\n")
return 1
if opts.direction not in ['asc', 'desc']:
sys.stderr.write("ERROR: invalid order direction\n")
return 1
# parse inputs
biom = None
rows = []
cols = []
data = []
try:
indata = sys.stdin.read() if opts.input == '-' else open(
opts.input, 'r').read()
if opts.format == 'biom':
try:
biom = json.loads(indata)
rows, cols, data = biom_to_matrix(biom, sig_stats=True)
except:
sys.stderr.write("ERROR: input BIOM data not correct format\n")
return 1
else:
rows, cols, data = tab_to_matrix(indata)
except:
sys.stderr.write("ERROR: unable to load input data\n")
return 1
# first we sort
if opts.order is not None:
rev_order = True if opts.direction == 'desc' else False
order_col = opts.order
if order_col > len(cols):
sys.stderr.write(
"ERROR: --order value is greater than number of columns in table\n"
)
order_col = order_col - 1
rd_merged = zip(rows, data)
rd_sorted = sorted(rd_merged,
key=lambda x: x[1][order_col],
reverse=rev_order)
rows, data = zip(*rd_sorted)
rows, data = list(rows), list(data)
# subselect rows
if opts.rows is not None:
subrow = opts.rows
rows = rows[:subrow]
data = data[:subrow]
if opts.cols is not None:
subcol = opts.cols
cols = cols[:subcol]
data = sub_matrix(data, subcol)
# output data
if biom and (opts.output == 'biom'):
# get list of new rows and columns with old indexes
br_ids = [r['id'] for r in biom['rows']]
bc_ids = [c['id'] for c in biom['columns']]
rindex = []
cindex = []
for r in rows:
try:
rindex.append(br_ids.index(r))
except:
pass
for c in cols:
try:
cindex.append(bc_ids.index(c))
except:
pass
# update biom
biom['id'] = biom['id'] + '_altered'
biom['data'] = sub_matrix(data, biom['shape'][1])
biom['rows'] = [biom['rows'][r] for r in rindex]
biom['columns'] = [biom['columns'][c] for c in cindex]
biom['shape'] = [len(biom['rows']), len(biom['columns'])]
biom['matrix_type'] = 'dense'
safe_print(json.dumps(biom) + '\n')
else:
safe_print("\t%s\n" % "\t".join(cols))
for i, d in enumerate(data):
safe_print("%s\t%s\n" % (rows[i], "\t".join(map(str, d))))
return 0