def silly_function(ui):
for c_value in ui.series(coloring_values):
sample_ids = sample_ids_from_metadata_description(open(mapping_fp, 'U'),
'%s:%s' % (coloring_header_name, c_value))
_headers, _data = filter_mapping_file(data, headers, sample_ids, True)
per_color_subject_values = list(set([row[subject_index] for row in _data]))
fd = open(join(output_path, 'color_by_'+c_value+'.txt'), 'w')
for s in ui.series(per_color_subject_values):
fd.write('%s\n' % s)
fd.close()
if not suppress_trajectory_files:
for s in ui.series(per_color_subject_values):
filename = join(output_path, s+'.txt')
if opts.verbose:
print 'Working on printing', filename
COMMAND_CALL = FILTER_CMD % (coords_fp, mapping_fp,
'%s:%s' % (subject_header_name, s), filename,
sorting_category)
o, e, r = qiime_system_call(COMMAND_CALL)
if opts.verbose and e:
print 'Error happened on filtering step: \n%s' % e
continue
COMMAND_CALL = CONVERSION_CMD % (filename, filename)
o, e, r = qiime_system_call(COMMAND_CALL)
if opts.verbose and e:
print 'Error happened on conversion step: \n%s' % e
continue # useless here but just in case
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if opts.submit_jobs and not opts.make_jobs:
option_parser.error('Must pass -m if passing -s. (Sorry about this, '
'it\'s for backwards-compatibility.)')
min_args = 2
if len(args) != min_args:
option_parser.error('Program requires <commands file> and '
'<job prefix>')
if (len(args[1]) > 10 or len(args[1]) == 0):
option_parser.error('job prefix must be 1-10 characters long')
if(not exists(opts.job_dir)):
try:
makedirs(opts.job_dir)
except OSError:
exit(" Jobs directory can not be created. "
"Check for permissions or file with the same name: %s\n"
% opts.job_dir)
commands = list(open(args[0]))
job_prefix = args[1]
if opts.mem_per_cpu:
mem_per_cpu = " --mem_per_cpu=" + opts.mem_per_cpu
else:
mem_per_cpu = ""
if opts.queue:
queue = " -p " + opts.queue
else:
queue = ""
if (opts.make_jobs):
filenames = make_jobs(
commands,
job_prefix,
opts.queue,
opts.job_dir)
else:
exit("Should we ever get here???")
if (opts.submit_jobs):
for f in filenames:
qiime_system_call("".join([
"sbatch",
queue,
" -J ", job_prefix,
mem_per_cpu,
" -o ", normpath(opts.job_dir), sep, job_prefix, "_%j.out",
" ", f
]), shell=True)
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if opts.submit_jobs and not opts.make_jobs:
option_parser.error('Must pass -m if passing -s. (Sorry about this, '
'it\'s for backwards-compatibility.)')
min_args = 2
if len(args) != min_args:
option_parser.error('Program requires <commands file> and '
'<job prefix>')
if (len(args[1]) > 10 or len(args[1]) == 0):
option_parser.error('job prefix must be 1-10 characters long')
if(not exists(opts.job_dir)):
try:
makedirs(opts.job_dir)
except OSError:
exit(" Jobs directory can not be created. "
"Check for permissions or file with the same name: %s\n"
% opts.job_dir)
commands = list(open(args[0]))
job_prefix = args[1]
if opts.mem_per_cpu:
mem_per_cpu = " --mem-per-cpu=" + opts.mem_per_cpu
else:
mem_per_cpu = ""
if opts.queue:
queue = " -p " + opts.queue
else:
queue = ""
if (opts.make_jobs):
filenames = make_jobs(
commands,
job_prefix,
opts.queue,
opts.job_dir)
else:
exit("Should we ever get here???")
if (opts.submit_jobs):
for f in filenames:
qiime_system_call("".join([
"sbatch",
queue,
" -J ", job_prefix,
mem_per_cpu,
" -o ", normpath(opts.job_dir), sep, job_prefix, "_%j.out",
" ", f
]), shell=True)
def call_commands_serially(commands,
status_update_callback,
logger,
close_logger_on_success=True):
"""Run list of commands, one after another """
logger.write("Executing commands.\n\n")
for c in commands:
for e in c:
status_update_callback('%s\n%s' % e)
logger.write('# %s command \n%s\n\n' % e)
stdout, stderr, return_value = qiime_system_call(e[1])
if return_value != 0:
msg = "\n\n*** ERROR RAISED DURING STEP: %s\n" % e[0] +\
"Command run was:\n %s\n" % e[1] +\
"Command returned exit status: %d\n" % return_value +\
"Stdout:\n%s\nStderr\n%s\n" % (stdout, stderr)
logger.write(msg)
logger.close()
raise WorkflowError(msg)
# in the no error case, we write commands' output to the log
# and also echo to this proc's stdout/stderr
else:
# write stdout and stderr to log file
logger.write("Stdout:\n%s\nStderr:\n%s\n" % (stdout, stderr))
# write stdout to stdout
if stdout:
print stdout
# write stderr to stderr
if stderr:
sys.stderr.write(stderr)
if close_logger_on_success:
logger.close()
def generate_random_password(min_len=8, max_len=12):
"""Returns a random alphanumeric password of random length.
Returns both unencrypted and encrypted password. Encryption is performed
via Apache's htpasswd command, using their custom MD5 algorithm.
Length will be randomly chosen from within the specified bounds
(inclusive).
"""
# Modified from
# http://code.activestate.com/recipes/59873-random-password-generation
chars = letters + digits
length = randint(min_len, max_len)
password = ''.join([choice(chars) for i in range(length)])
# This is hackish but should work for now...
stdout, stderr, ret_val = qiime_system_call('htpasswd -nbm foobarbaz %s' %
password)
if ret_val != 0:
raise ValueError("Error executing htpasswd command. Do you have this "
"command on your machine?")
# Will be in the form foobarbaz:<encrypted password>
encrypted_password = stdout.strip().split('foobarbaz:', 1)[1]
return password, encrypted_password
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
which('mothur'),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), 'mothur.log')
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def copy_support_files(file_path):
"""Copy the support files to a named destination
file_path: path where you want the support files to be copied to
Will raise EmperorSupportFilesError if a problem is found whilst trying to
copy the files.
"""
file_path = join(file_path, 'emperor_required_resources')
if exists(file_path) == False:
create_dir(file_path, False)
# shutil.copytree does not provide an easy way to copy the contents of a
# directory into another existing directory, hence the system call.
# use double quotes for the paths to escape any invalid chracter(s)/spaces
cmd = 'cp -R "%s/"* "%s"' % (get_emperor_support_files_dir(),
abspath(file_path))
cmd_o, cmd_e, cmd_r = qiime_system_call(cmd)
if cmd_e:
raise EmperorSupportFilesError, "Error found whilst trying to copy " +\
"the support files:\n%s\n Could not execute: %s" % (cmd_e, cmd)
return
def _iter_ids_over_system_call(cmd_fmt, sample_ids, opts):
"""Iteratively execute a system call over sample IDs
Parameters
----------
cmd_fmt : str
The format of the command to execute. It is expected that there
is a single string format to be done and it should take a sample
ID
sample_ids : Iterable of str
A list of sample IDs of interest
Returns
-------
dict
A dict containing each sample ID and any errors observed or None if
no error was observed for the sample. {str: str or None}
"""
results = {}
for id_ in sample_ids:
cmd = cmd_fmt % {'result_path': _result_path(opts, id_), 'id': id_}
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = stderr.splitlines()
results[id_] = 'FAILED (%s): %s' % (msg[-1] if msg else '', cmd)
else:
results[id_] = None
return results
def call_commands_serially(commands,
status_update_callback,
logger,
close_logger_on_success=True):
"""Run list of commands, one after another """
logger.write("Executing commands.\n\n")
for c in commands:
for e in c:
status_update_callback('%s\n%s' % e)
logger.write('# %s command \n%s\n\n' % e)
stdout, stderr, return_value = qiime_system_call(e[1])
if return_value != 0:
msg = "\n\n*** ERROR RAISED DURING STEP: %s\n" % e[0] +\
"Command run was:\n %s\n" % e[1] +\
"Command returned exit status: %d\n" % return_value +\
"Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
logger.write(msg)
logger.close()
raise WorkflowError, msg
# in the no error case, we write commands' output to the log
# and also echo to this proc's stdout/stderr
else:
# write stdout and stderr to log file
logger.write("Stdout:\n%s\nStderr:\n%s\n" % (stdout,stderr))
# write stdout to stdout
if stdout:
print stdout
# write stderr to stderr
if stderr:
sys.stderr.write(stderr)
if close_logger_on_success: logger.close()
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
which("mothur"),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), "mothur.log")
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(" ")[1].strip("v.")
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def copy_support_files(file_path):
"""Copy the support files to a named destination
file_path: path where you want the support files to be copied to
Will raise EmperorSupportFilesError if a problem is found whilst trying to
copy the files.
"""
file_path = join(file_path, "emperor_required_resources")
if exists(file_path) == False:
create_dir(file_path, False)
# shutil.copytree does not provide an easy way to copy the contents of a
# directory into another existing directory, hence the system call.
# use double quotes for the paths to escape any invalid chracter(s)/spaces
cmd = 'cp -R "%s/"* "%s"' % (get_emperor_support_files_dir(), abspath(file_path))
cmd_o, cmd_e, cmd_r = qiime_system_call(cmd)
if cmd_e:
raise EmperorSupportFilesError, "Error found whilst trying to copy " + "the support files:\n%s\n Could not execute: %s" % (
cmd_e,
cmd,
)
return
def _iter_ids_over_system_call(cmd_fmt, sample_ids, opts):
"""Iteratively execute a system call over sample IDs
Parameters
----------
cmd_fmt : str
The format of the command to execute. It is expected that there
is a single string format to be done and it should take a sample
ID
sample_ids : Iterable of str
A list of sample IDs of interest
Returns
-------
dict
A dict containing each sample ID and any errors observed or None if
no error was observed for the sample. {str: str or None}
"""
results = {}
for id_ in sample_ids:
cmd = cmd_fmt % {'result_path': _result_path(opts, id_),
'id': id_}
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = stderr.splitlines()
results[id_] = 'FAILED (%s): %s' % (msg[-1] if msg else '', cmd)
else:
results[id_] = None
return results
def run_command(cmd):
stdout, stderr, ret_val = qiime_system_call(cmd)
if ret_val != 0:
raise ExternalCommandFailedError("The command '%s' failed with exit "
"status %d.\n\nStdout:\n\n%s\n\n"
"Stderr:\n\n%s\n" % (cmd,
ret_val, stdout, stderr))
def make_line_plot(
dir_path, data_file_link, background_color, label_color, xy_coords,
props, x_len=8, y_len=4, draw_axes=False, generate_eps=True):
""" Write a line plot
xy_coords: a dict of form
{series_label:([x data], [y data], point_marker, color)}
(code adapted from Micah Hamady's code)
"""
rc('font', size='8')
rc('axes', linewidth=.5, edgecolor=label_color)
rc('axes', labelsize=8)
rc('xtick', labelsize=8)
rc('ytick', labelsize=8)
fig, ax = plt.subplots(figsize=(x_len, y_len))
mtitle = props.get("title", "Groups")
x_label = props.get("xlabel", "X")
y_label = props.get("ylabel", "Y")
ax.set_title('%s' % mtitle, fontsize='10', color=label_color)
ax.set_xlabel(x_label, fontsize='8', color=label_color)
ax.set_ylabel(y_label, fontsize='8', color=label_color)
sorted_keys = sorted(xy_coords.keys())
for s_label in sorted_keys:
s_data = xy_coords[s_label]
c = s_data[3]
m = s_data[2]
ax.plot(s_data[0], s_data[1], c=c, marker=m, label=s_label,
linewidth=.1, ms=5, alpha=1.0)
fp = FontProperties()
fp.set_size('8')
ax.legend(prop=fp, loc=0)
img_name = 'scree_plot.png'
fig.savefig(
os.path.join(dir_path,
img_name),
dpi=80,
facecolor=background_color)
# Create zipped eps files
eps_link = ""
if generate_eps:
eps_img_name = str('scree_plot.eps')
fig.savefig(os.path.join(dir_path, eps_img_name), format='eps')
out, err, retcode = qiime_system_call(
"gzip -f " + os.path.join(dir_path, eps_img_name))
eps_link = DOWNLOAD_LINK % ((os.path.join(data_file_link,
eps_img_name) +
".gz"), "Download Figure")
return os.path.join(data_file_link, img_name), eps_link
def get_emperor_library_version():
"""Get Emperor version and the git SHA + current branch (if applicable)"""
emperor_dir = get_emperor_project_dir()
emperor_version = emperor_library_version
# more information could be retrieved following this pattern
sha_cmd = "git --git-dir %s/.git rev-parse HEAD" % (emperor_dir)
sha_o, sha_e, sha_r = qiime_system_call(sha_cmd)
git_sha = sha_o.strip()
branch_cmd = "git --git-dir %s/.git rev-parse --abbrev-ref HEAD" % (emperor_dir)
branch_o, branch_e, branch_r = qiime_system_call(branch_cmd)
git_branch = branch_o.strip()
# validate the output from both command calls
if is_valid_git_refname(git_branch) and is_valid_git_sha1(git_sha):
return "%s, %s@%s" % (emperor_version, git_branch, git_sha[0:7])
else:
return "%s" % emperor_version
def submit_jobs(path_to_cluster_jobs, jobs_fp, job_prefix):
""" Submit the jobs to the queue using cluster_jobs.py
"""
cmd = '%s -ms %s %s' % (path_to_cluster_jobs, jobs_fp, job_prefix)
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = "\n\n*** Could not start parallel jobs. \n" +\
"Command run was:\n %s\n" % cmd +\
"Command returned exit status: %d\n" % return_value +\
"Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
raise RuntimeError, msg
def get_emperor_library_version():
"""Get Emperor version and the git SHA + current branch (if applicable)"""
emperor_dir = get_emperor_project_dir()
emperor_version = emperor_library_version
# more information could be retrieved following this pattern
sha_cmd = 'git --git-dir %s/.git rev-parse HEAD' % (emperor_dir)
sha_o, sha_e, sha_r = qiime_system_call(sha_cmd)
git_sha = sha_o.strip()
branch_cmd = 'git --git-dir %s/.git rev-parse --abbrev-ref HEAD' %\
(emperor_dir)
branch_o, branch_e, branch_r = qiime_system_call(branch_cmd)
git_branch = branch_o.strip()
# validate the output from both command calls
if is_valid_git_refname(git_branch) and is_valid_git_sha1(git_sha):
return '%s, %s@%s' % (emperor_version, git_branch, git_sha[0:7])
else:
return '%s' % emperor_version
def call_cmd(cmd, HALT_EXEC):
if HALT_EXEC:
print cmd
exit(0)
else:
stdout, stderr, exit_status = qiime_system_call(cmd)
if exit_status != 0:
print "indexdb_rna failed!\nSTDOUT\n%s\nSTDERR\n%s\n" \
% (stdout, stderr)
exit(1)
return cmd
def run_commands(output_dir,commands,run_id,submit_jobs,keep_temp,queue_name):
"""
"""
job_fps, paths_to_remove = write_job_files(output_dir,commands,run_id,queue_name)
# Call the jobs
if submit_jobs:
for job_fp in job_fps:
qiime_system_call(' '.join(['qsub', job_fp]))
# clean up the shell scripts that were created
if not keep_temp:
for p in paths_to_remove:
try:
# p is file
remove(p)
except OSError:
# p is directory
rmtree(p)
return
def run_commands(output_dir, commands, run_id, submit_jobs, keep_temp,
queue_name):
"""
"""
job_fps, paths_to_remove = write_job_files(output_dir, commands, run_id,
queue_name)
# Call the jobs
if submit_jobs:
for job_fp in job_fps:
qiime_system_call(' '.join(['qsub', job_fp]))
# clean up the shell scripts that were created
if not keep_temp:
for p in paths_to_remove:
try:
# p is file
remove(p)
except OSError:
# p is directory
rmtree(p)
return
def _submit_jobs(self, jobs_fp, job_prefix):
""" Submit the jobs to the queue using cluster_jobs.py
"""
cmd = '%s -ms %s %s' % (self._cluster_jobs_fp, jobs_fp, job_prefix)
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = "\n\n*** Could not start parallel jobs. \n" +\
"Command run was:\n %s\n" % cmd +\
"Command returned exit status: %d\n" % return_value +\
"Stdout:\n%s\nStderr\n%s\n" % (stdout, stderr)
raise RuntimeError(msg)
# Leave this comments in as they're useful for debugging.
# print 'Return value: %d\n' % return_value
# print 'STDOUT: %s\n' % stdout
# print 'STDERR: %s\n' % stderr
return stdout, stderr, return_value
def _submit_jobs(self, jobs_fp, job_prefix):
""" Submit the jobs to the queue using cluster_jobs.py
"""
cmd = "%s -ms %s %s" % (self._cluster_jobs_fp, jobs_fp, job_prefix)
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = (
"\n\n*** Could not start parallel jobs. \n"
+ "Command run was:\n %s\n" % cmd
+ "Command returned exit status: %d\n" % return_value
+ "Stdout:\n%s\nStderr\n%s\n" % (stdout, stderr)
)
raise RuntimeError, msg
# Leave this comments in as they're useful for debugging.
# print 'Return value: %d\n' % return_value
# print 'STDOUT: %s\n' % stdout
# print 'STDERR: %s\n' % stderr
return stdout, stderr, return_value
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if (opts.suppress_unit_tests):
option_parser.error("Tests, suppresed. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of Evident's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root,name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % '\n'.join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due "+\
"to missing external applications.\nDepending on the Evident features "+\
"you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
qiime_test_data_dir_exists = True
# In case there were no failures of any type, exit with a return code of 0
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and\
qiime_test_data_dir_exists):
return_code = 0
return return_code
def submit_fasta_and_split_lib(data_access,fasta_files,metadata_study_id,
input_dir):
"""
FASTA Loading: This function takes the fasta filenames and using that
path, determines the location of the split-library and picked-otu files.
Once file locations have been determined, it moves the files to the DB
machine and load the files into the DB.
"""
# get DB connection and cursor
con = data_access.getSFFDatabaseConnection()
cur = con.cursor()
# check if study exists
study_id_exists=data_access.checkIfStudyIdExists(metadata_study_id)
print "Study ID exists: " + str(study_id_exists)
# get temp filename
alphabet = "ABCDEFGHIJKLMNOPQRSTUZWXYZ"
alphabet += alphabet.lower()
alphabet += "01234567890"
random_fname=''.join([choice(alphabet) for i in range(10)])
tmp_filename ='_'+random_fname+'_'+strftime("%Y_%m_%d_%H_%M_%S")
# get fasta filenames
fasta_filenames=fasta_files.split(',')
seq_run_id=0
analysis_id=0
split_lib_input_checksums=[]
### by disabling constraints you can speed up loading as well, but shouldn't
### be necessary
#valid = data_access.disableTableConstraints()
#print "Disabled table constraints"
# split the fasta filenames and determine filepaths
for fasta_fname in fasta_filenames:
input_fname, input_ext = splitext(split(fasta_fname)[-1])
input_basename, input_ext = splitext(fasta_fname)
# define analysis notes
analysis_notes=split(input_basename)[0]
# get md5 for raw fasta files
fasta_md5 = safe_md5(open(fasta_fname)).hexdigest()
print 'MD5 is: %s' % str(fasta_md5)
# create an analysis row in analysis table
if analysis_id==0:
analysis_id=data_access.createAnalysis(metadata_study_id)
# check if fasta info already loaded
fasta_exists=data_access.checkIfSFFExists(fasta_md5)
print 'fasta in database? %s' % str(fasta_exists)
# if fasta info not loaded, then insert into DB
if not fasta_exists:
if seq_run_id==0:
seq_run_id=data_access.createSequencingRun(True,'FASTA',
None,seq_run_id)
# get sequence count
count_seqs_cmd="grep '^>' %s | wc -l" % (fasta_fname)
o,e,r = qiime_system_call(count_seqs_cmd)
seq_counts = o.strip()
# add fasta info
valid=data_access.addSFFFileInfo(True,input_fname,
seq_counts,
None,
None,
None,
None,
None,
None,
fasta_md5,seq_run_id)
else:
seq_run_id=data_access.getSeqRunIDUsingMD5(fasta_md5)
print 'sequence_run_id is: %s' % str(seq_run_id)
# get md5 sum for input to split-libraries
split_lib_input_md5sum=safe_md5(MD5Wrap(fasta_filenames)).hexdigest()
print split_lib_input_md5sum
print 'Finished loading the processed FASTA data!'
print 'Run ID: %s' % seq_run_id
print 'Analysis ID: %s' % analysis_id
# update analysis table with seq_run_id
valid=data_access.updateAnalysisWithSeqRunID(True,analysis_id,seq_run_id)
if not valid:
raise ValueError, 'Error: Unable to append SEQ_RUN_ID into ANALYSIS table!'
return analysis_id,input_dir,seq_run_id,split_lib_input_md5sum
def make_interactive_scatter(plot_label,
dir_path,
data_file_link,
background_color,
label_color,
sample_location,
alpha,
xy_coords,
props,
x_len=8,
y_len=4,
size=10,
draw_axes=False,
generate_eps=True):
"""Write interactive plot
xy_coords: a dict of form {series_label:([x data], [y data], \
[xy point label],[color])}
"""
my_axis = None
rc('font', size='8')
rc('patch', linewidth=0)
rc('axes', linewidth=.5, edgecolor=label_color)
rc('axes', labelsize=8)
rc('xtick', labelsize=8, color=label_color)
rc('ytick', labelsize=8, color=label_color)
sc_plot = draw_scatterplot(props, xy_coords, x_len, y_len, size,
background_color, label_color, sample_location,
alpha)
mtitle = props.get("title", "Groups")
x_label = props.get("xlabel", "X")
y_label = props.get("ylabel", "Y")
title('%s' % mtitle, fontsize='10', color=label_color)
xlabel(x_label, fontsize='8', color=label_color)
ylabel(y_label, fontsize='8', color=label_color)
show()
if draw_axes:
axvline(linewidth=.5, x=0, color=label_color)
axhline(linewidth=.5, y=0, color=label_color)
if my_axis is not None:
axis(my_axis)
img_name = x_label[0:3] + '_vs_' + y_label[0:3] + '_plot.png'
savefig(os.path.join(dir_path, img_name),
dpi=80,
facecolor=background_color)
# Create zipped eps files
eps_link = ""
if generate_eps:
eps_img_name = str(x_label[0:3] + 'vs' + y_label[0:3] + 'plot.eps')
savefig(os.path.join(dir_path, eps_img_name), format='eps')
out, err, retcode = qiime_system_call(
"gzip -f " + os.path.join(dir_path, eps_img_name))
eps_link = DOWNLOAD_LINK % (
(os.path.join(data_file_link, eps_img_name) + ".gz"),
"Download Figure")
all_cids, all_xcoords, all_ycoords = transform_xy_coords(
xy_coords, sc_plot)
xmap, img_height, img_width = generate_xmap(x_len, y_len, all_cids,
all_xcoords, all_ycoords)
points_id = plot_label + x_label[2:3] + y_label[2:3]
return IMG_MAP_SRC % (os.path.join(data_file_link, img_name), points_id,
img_width, img_height), MAP_SRC % \
(points_id, ''.join(xmap)), eps_link
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
if opts.suppress_unit_tests and opts.suppress_script_tests and opts.suppress_script_usage_tests:
option_parser.error("You're suppressing all three test types. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile("OK\s*$")
application_not_found_pattern = re.compile("ApplicationNotFoundError")
python_name = "python"
bad_tests = []
missing_application_tests = []
# Run through all of QIIME's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith("test_") and name.endswith(".py"):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith("test_") and fn.endswith(".py"):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = "%s %s -v" % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
bad_scripts = []
if not opts.suppress_script_tests:
# Run through all of QIIME's scripts, and pass -h to each one. If the
# resulting stdout does not being with the Usage text, that is an
# indicator of something being wrong with the script. Issues that would
# cause that are bad import statements in the script, SyntaxErrors, or
# other failures prior to running qiime.util.parse_command_line_parameters.
try:
scripts_dir = get_qiime_scripts_dir()
script_directory_found = True
except AssertionError:
script_directory_found = False
if script_directory_found:
script_names = []
script_names = glob("%s/*py" % scripts_dir)
script_names.sort()
for script_name in script_names:
script_good_pattern = re.compile("^Usage: %s" % split(script_name)[1])
print "Testing %s." % script_name
command = "%s %s -h" % (python_name, script_name)
stdout, stderr, return_value = qiime_system_call(command)
if not script_good_pattern.search(stdout):
bad_scripts.append(script_name)
num_script_usage_example_failures = 0
qiime_test_data_dir = qiime_config["qiime_test_data_dir"]
if not opts.suppress_script_usage_tests and qiime_test_data_dir != None:
# Run the script usage testing functionality
script_usage_result_summary, num_script_usage_example_failures = run_script_usage_tests(
qiime_test_data_dir=qiime_test_data_dir,
qiime_scripts_dir=qiime_config["qiime_scripts_dir"],
working_dir=qiime_config["temp_dir"],
verbose=True,
tests=None, # runs all
failure_log_fp=None,
force_overwrite=True,
)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % "\n".join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due " + "to missing external applications.\nDepending on the QIIME features " + "you plan to use, this may not be critical.\n%s" % "\n".join(
missing_application_tests
)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not opts.suppress_script_tests:
print "\nBasic script test result summary\n--------------------------------\n"
if not script_directory_found:
print "Critical error: Failed to test scripts because the script directory could not be found.\n The most likely explanation for this failure is that you've installed QIIME using setup.py, and forgot to specify the qiime_scripts_dir in your qiime_config file. This value shoud be set either to the directory you provided for --install-scripts, or /usr/local/bin if no value was provided to --install-scripts."
else:
if bad_scripts:
print "Failed the following basic script tests.\n%s" % "\n".join(bad_scripts)
else:
print "All basic script tests passed successfully.\n"
qiime_test_data_dir_exists = True
if not opts.suppress_script_usage_tests:
if qiime_test_data_dir:
print "\nScript usage test result summary\n------------------------------------\n"
print script_usage_result_summary
else:
print "\nCould not run script usage tests because qiime_test_data_dir is not defined in your qiime_config."
qiime_test_data_dir_exists = False
print ""
# If any of the unit tests, script tests, or script usage tests fail, or if
# we have any missing application errors or a missing QIIME test data dir
# if script usage tests weren't suppressed, use return code 1 (as python's
# unittest module does to indicate one or more failures).
return_code = 1
if (
len(bad_tests) == 0
and len(missing_application_tests) == 0
and len(bad_scripts) == 0
and num_script_usage_example_failures == 0
and qiime_test_data_dir_exists
):
return_code = 0
return return_code
def create_personal_results(mapping_fp,
distance_matrix_fp,
collated_dir_fp,
output_fp, prefs_fp,
personal_id_field,
personal_ids=None,
column_title=None,
individual_titles=None):
map_as_list, header, comments = parse_mapping_file(open(mapping_fp, 'U'))
try:
personal_id_index = header.index(personal_id_field)
except ValueError:
raise ValueError("personal id field (%s) is not a mapping file column header" %
personal_id_field)
if column_title == None:
header.append('Self')
else:
header.append(column_title)
if personal_ids == None:
personal_ids = get_personal_ids(map_as_list, personal_id_index)
else:
for id in personal_ids.split(','):
if id not in get_personal_ids(map_as_list, personal_id_index):
raise ValueError('%s is not an id in the mapping file.' %id)
personal_ids = personal_ids.split(',')
output_directories = []
makedirs(output_fp)
for person_of_interest in personal_ids:
makedirs(join(output_fp, person_of_interest))
pcoa_dir = join(output_fp, person_of_interest, "beta_diversity")
rarefaction_dir = join(output_fp, person_of_interest, "alpha_rarefaction")
output_directories.append(pcoa_dir)
output_directories.append(rarefaction_dir)
personal_mapping_file_fp = join(output_fp, person_of_interest, "mapping_file.txt")
html_fp = join(output_fp, person_of_interest, "index.html")
create_personal_mapping_file(map_as_list,
header,
comments,
person_of_interest,
personal_mapping_file_fp,
personal_id_index,
individual_titles)
create_index_html(person_of_interest, html_fp)
cmd = "make_rarefaction_plots.py -i %s -m %s -p %s -o %s" % (collated_dir_fp,
personal_mapping_file_fp,
prefs_fp,
rarefaction_dir)
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
cmd = "make_3d_plots.py -m %s -p %s -i %s -o %s" % (personal_mapping_file_fp,
prefs_fp,
distance_matrix_fp,
pcoa_dir)
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
return output_directories
def run_script_usage_tests(qiime_test_data_dir,
qiime_scripts_dir,
working_dir,
verbose=False,
tests=None,
failure_log_fp=None,
force_overwrite=False):
""" Test script_usage examples when test data is present in qiime_test_data_dir
These tests are currently used with the qiime_test_data repository, which can
be found at https://github.com/qiime-dev/qiime_test_data
Returns a result summary string and the number of script usage
examples (i.e. commands) that failed.
"""
# process input filepaths and directories
qiime_test_data_dir = abspath(qiime_test_data_dir)
working_dir = join(working_dir,'script_usage_tests')
if force_overwrite and exists(working_dir):
rmtree(working_dir)
if failure_log_fp != None:
failure_log_fp = abspath(failure_log_fp)
if tests == None:
tests = [split(d)[1] for d in glob('%s/*' % qiime_test_data_dir) if isdir(d)]
if verbose:
print 'Tests to run:\n %s' % ' '.join(tests)
addsitedir(qiime_scripts_dir)
failed_tests = []
warnings = []
total_tests = 0
for test in tests:
# import the usage examples - this is possible because we added
# qiime_scripts_dir to the PYTHONPATH above
script_fn = '%s/%s.py' % (qiime_scripts_dir,test)
script = __import__(test)
usage_examples = script.script_info['script_usage']
if verbose:
print 'Testing %d usage examples from: %s' % (len(usage_examples),script_fn)
# init the test environment
test_input_dir = '%s/%s' % (qiime_test_data_dir,test)
test_working_dir = '%s/%s' % (working_dir,test)
copytree(test_input_dir,test_working_dir)
chdir(test_working_dir)
# remove pre-exisitng output files if any
try:
script_usage_output_to_remove = script.script_info['script_usage_output_to_remove']
except KeyError:
script_usage_output_to_remove = []
for e in script_usage_output_to_remove:
rmtree(e.replace('$PWD',getcwd()),ignore_errors=True)
if verbose:
print ' Running tests in: %s' % getcwd()
print ' Tests:'
for usage_example in usage_examples:
if '%prog' not in usage_example[2]:
warnings.append('%s usage examples do not all use %%prog to represent the command name. You may not be running the version of the command that you think you are!' % test)
cmd = usage_example[2].replace('%prog',script_fn)
if verbose:
print ' %s' % cmd
stdout, stderr, return_value = qiime_system_call(cmd)
total_tests += 1
if return_value != 0:
failed_tests.append((cmd, stdout, stderr, return_value))
if verbose:
print ''
if failure_log_fp:
failure_log_f = open(failure_log_fp,'w')
if len(failed_tests) == 0:
failure_log_f.write('All script interface tests passed.\n')
else:
i = 0
for cmd, stdout, stderr, return_value in failed_tests:
failure_log_f.write('**Failed test %d:\n%s\n\nReturn value: %d\n\nStdout:\n%s\n\nStderr:\n%s\n\n' % (i,cmd,return_value, stdout, stderr))
failure_log_f.close()
if warnings:
print 'Warnings:'
for warning in warnings:
print ' ' + warning
print ''
result_summary = 'Ran %d commands to test %d scripts. %d of these commands failed.' % (total_tests,len(tests),len(failed_tests))
if len(failed_tests) > 0:
failed_scripts = set([split(e[0].split()[0])[1] for e in failed_tests])
result_summary += '\nFailed scripts were: %s' % " ".join(failed_scripts)
if failure_log_fp:
result_summary += "\nFailures are summarized in %s" % failure_log_fp
rmtree(working_dir)
return result_summary, len(failed_tests)
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
chuck_norris_joke = opts.chuck_norris_joke
coordinates_fp = opts.coordinates_fp
mapping_file_fp = opts.mapping_file_fp
category_header_name = opts.category
output_fp = opts.output_fp
# have a swell day Yoshiki from the future
if chuck_norris_joke:
o, e, _ = qiime_system_call('curl http://api.icndb.com/jokes/random')
exec 'joke = %s' % o.strip()
print joke['value']['joke']
exit(0)
coords_headers, coords_data, coords_eigenvalues, coords_percents =\
parse_coords(open(coordinates_fp, 'U'))
mapping_data, mapping_headers, _ = parse_mapping_file(open(mapping_file_fp, 'U'))
category_header_index = mapping_headers.index(category_header_name)
category_names = list(set([line[category_header_index]
for line in mapping_data]))
main_figure = plt.figure()
main_axes = main_figure.add_subplot(1, 1, 1, axisbg='black')
plt.xlabel('PC1')
plt.ylabel('PC2')
main_axes.tick_params(axis='y', colors='none')
main_axes.tick_params(axis='x', colors='none')
# sort the data!!! that way you can match make_3d_plots.py
sorted_categories = natsort(category_names)
colors_used = []
for index, category in enumerate(sorted_categories):
sample_ids_list = [line[0] for line in mapping_data if line[category_header_index] == category]
qiime_color = get_qiime_hex_string_color(index)
if len(sample_ids_list) < 3:
continue
colors_used.append(qiime_color)
indices = [coords_headers.index(sample_id) for sample_id in sample_ids_list]
points = coords_data[indices, :2]# * coords_percents[:2]
hull = ConvexHull(points)
main_axes.plot(points[:,0], points[:,1], 'o', color=qiime_color)
for simplex in hull.simplices:
main_axes.plot(points[simplex,0], points[simplex,1], 'w-')
main_axes.plot(points[hull.vertices,0], points[hull.vertices,1], '--', lw=2, color=qiime_color)
# plt.plot(points[hull.vertices[0],0], points[hull.vertices[0],1], '--', color=qiime_color)
# plt.show()
main_figure.savefig(output_fp)
name = splitext(output_fp)[0]
extension = splitext(output_fp)[1].replace('.', '')
make_legend(sorted_categories, colors_used, 0, 0, 'black', 'white', name,
extension, 80)
def make_interactive_scatter(plot_label, dir_path, data_file_link,
background_color, label_color, sample_location,
alpha, xy_coords,
props, x_len=8, y_len=4, size=10,
draw_axes=False, generate_eps=True):
"""Write interactive plot
xy_coords: a dict of form {series_label:([x data], [y data], \
[xy point label],[color])}
"""
my_axis = None
rc('font', size='8')
rc('patch', linewidth=0)
rc('axes', linewidth=.5, edgecolor=label_color)
rc('axes', labelsize=8)
rc('xtick', labelsize=8, color=label_color)
rc('ytick', labelsize=8, color=label_color)
sc_plot = draw_scatterplot(props, xy_coords, x_len, y_len, size,
background_color, label_color, sample_location,
alpha)
mtitle = props.get("title", "Groups")
x_label = props.get("xlabel", "X")
y_label = props.get("ylabel", "Y")
ax = plt.gca()
fig = ax.figure
ax.set_title('%s' % mtitle, fontsize='10', color=label_color)
ax.set_xlabel(x_label, fontsize='8', color=label_color)
ax.set_ylabel(y_label, fontsize='8', color=label_color)
if draw_axes:
ax.axvline(linewidth=.5, x=0, color=label_color)
ax.axhline(linewidth=.5, y=0, color=label_color)
if my_axis is not None:
ax.axis(my_axis)
img_name = x_label[0:3] + '_vs_' + y_label[0:3] + '_plot.png'
fig.savefig(os.path.join(dir_path, img_name),
dpi=80, facecolor=background_color)
# Create zipped eps files
eps_link = ""
if generate_eps:
eps_img_name = str(x_label[0:3] + 'vs' + y_label[0:3] + 'plot.eps')
fig.savefig(os.path.join(dir_path, eps_img_name), format='eps')
out, err, retcode = qiime_system_call(
"gzip -f " + os.path.join(dir_path, eps_img_name))
eps_link = DOWNLOAD_LINK % ((os.path.join(data_file_link, eps_img_name)
+ ".gz"), "Download Figure")
all_cids, all_xcoords, all_ycoords = transform_xy_coords(
xy_coords, sc_plot)
xmap, img_height, img_width = generate_xmap(
x_len, y_len, all_cids, all_xcoords,
all_ycoords)
points_id = plot_label + x_label[2:3] + y_label[2:3]
return IMG_MAP_SRC % (os.path.join(data_file_link, img_name), points_id,
img_width, img_height), MAP_SRC % \
(points_id, ''.join(xmap)), eps_link
def get_cluster_ratio(fasta_seqs, min_difference_in_clusters):
"""
Uses uclust to calculate cluster ratio
cluster_ratio =
num_of_seq_in_cluster_with_max_seq
divided by
num_of_seq_in cluster_with_second_higest_seq
Parameters
----------
fasta_seqs: list
list of fasta sequences
min_difference_in_clusters: float
percent identity threshold for cluster formation
Returns
----------
cluster_ratio: float
cluster ratio of the sequences using uclust
cluster_ratio =
num_of_seq_in_cluster_with_max_seq /
num_of_seq_in cluster_with_second_higest_seq
"""
cluster_percent_id = min_difference_in_clusters
temp_dir = get_qiime_temp_dir()
fd_uc, uclust_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
close(fd_uc)
fd_fas, fasta_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
close(fd_fas)
with open(fasta_tempfile_name, 'w') as fasta_tempfile:
fasta_tempfile.write(fasta_seqs)
fasta_tempfile.close()
count = 0
command = "uclust --usersort --input {} --uc {} --id 0.98".format(
fasta_tempfile_name, uclust_tempfile_name)
# In the function, I am calling uclust a large number of times.
# Initially I was using from bfillings.get_clusters_from_fasta_filepath
# but due to issue (biocore/bfillingss#31), I have temporarily
# reverted to qiime_system_call.
count_lookup = defaultdict(int)
qiime_system_call(command)
uclust_tempfile = open(uclust_tempfile_name, 'r')
for line in uclust_tempfile:
if search(r'^C', line):
pieces = line.split('\t')
count_lookup[pieces[1]] += int(pieces[2])
count += 1
uclust_tempfile.close()
files_to_be_removed = list()
files_to_be_removed.append(uclust_tempfile_name)
remove_files(files_to_be_removed)
sorted_counts_in_clusters = sorted(
count_lookup.iteritems(),
key=lambda x: x[1], reverse=True)
try:
max_cluster_count = \
float(str(sorted_counts_in_clusters[0][1]))
second_cluster_count = \
float(str(sorted_counts_in_clusters[1][1]))
return max_cluster_count / second_cluster_count
except IndexError:
return 1
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if (opts.suppress_unit_tests):
option_parser.error("Tests, suppresed. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of Evident's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % '\n'.join(
bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due "+\
"to missing external applications.\nDepending on the Evident features "+\
"you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
qiime_test_data_dir_exists = True
# In case there were no failures of any type, exit with a return code of 0
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and\
qiime_test_data_dir_exists):
return_code = 0
return return_code
# just indicate whether it's installed or not
try:
import gdata
except ImportError:
gdata_installed = "Not installed."
else:
gdata_installed = "Installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
break
script_info = {}
def __call__(self,
input_fp,
output_dir,
params,
job_prefix=None,
poll_directly=False,
suppress_submit_jobs=False):
""" """
## Generate a list of files and directories that will need to be cleaned up
self.files_to_remove = []
# Perform any method-specific setup. This should prevent the need to
# overwrite __call__
self._call_initialization(input_fp,
output_dir,
params,
job_prefix,
poll_directly,
suppress_submit_jobs)
# split the input filepath into directory and filename, base filename and
# extension for use in naming other files
try:
input_dir, input_fn = split(input_fp)
input_file_basename, input_ext = splitext(input_fn)
except AttributeError:
## THIS IS AWFUL - SOME OF THE SCRIPTS PASS A LIST, SO THE
## PREVIOUS BLOCK WON'T WORK... WHAT DO WE WANT TO DO?
input_dir, input_fn = split(input_fp[0])
input_file_basename, input_ext = splitext(input_fn)
# Allow the user to override the default job_prefix (defined by the
# base classes)
if job_prefix is None:
job_prefix = self._get_random_job_prefix(self._job_prefix)
# A temporary output directory is created in output_dir named
# job_prefix. Output files are then moved from the temporary
# directory to the output directory when they are complete,
# allowing a poller to detect when runs complete by the presence
# of their output files.
working_dir = join(output_dir,job_prefix)
try:
makedirs(working_dir)
self.files_to_remove.append(working_dir)
except OSError:
# working dir already exists
pass
# Split the input file into the individual job input files. Add the
# individual job files to the files_to_remove list
input_fps, remove_input_on_completion = self._input_splitter(
input_fp,
params,
self._jobs_to_start,
job_prefix,
working_dir)
if remove_input_on_completion:
self.files_to_remove += input_fps
# Perform any method-specific setup (e.g., formatting a BLAST database)
self._precommand_initiation(input_fp,output_dir,working_dir,params)
# Generate the list of commands to be pushed out to workers
# and the list of output files generated by each job.
commands, job_result_filepaths = self._get_job_commands(input_fps,
output_dir,
params,
job_prefix,
working_dir)
self.files_to_remove += \
self._identify_files_to_remove(job_result_filepaths,params)
# Generate the output clean-up files
merge_map_filepath, deletion_list_filepath, expected_files_filepath =\
self._initialize_output_cleanup_files(job_result_filepaths,
output_dir,
working_dir,
input_file_basename,
params)
# Set up poller apparatus if the user does not suppress polling
if not self._suppress_polling:
poller_command = self._initiate_polling(job_result_filepaths,
working_dir,
poll_directly,
merge_map_filepath,
deletion_list_filepath,
expected_files_filepath)
# If the poller should be run in the same way as the other commands
# (rather than by the current process), add it to the list of commands
if not poll_directly:
commands.append(poller_command)
# Build the filepath for the 'jobs script'. Add that file to the
# files_to_remove list.
jobs_fp = join(working_dir,job_prefix + 'jobs.txt')
self._write_jobs_file(commands,jobs_fp)
self.files_to_remove.append(jobs_fp)
# submit the jobs file using cluster_jobs, if not suppressed by the
# user
if not suppress_submit_jobs:
stdout, stderr, return_value = self._submit_jobs(
jobs_fp=jobs_fp, job_prefix=job_prefix)
# If the poller is going to be run by the current process,
# start polling
if poll_directly:
# IMPORTANT: the following line MUST use qiime_system_call()
# instead of subprocess.call, .check_call, or .check_output in case
# we are invoked in a child process with PIPEs (a deadlock will
# occur otherwise). This can happen if this code is tested by
# all_tests.py, for example.
stdout, stderr, return_value = qiime_system_call(poller_command)
if return_value != 0:
print '**Error occuring when calling the poller directly. '+\
'Jobs may have been submitted, but are not being polled.'
print stderr
print poller_command
exit(-1)
self.files_to_remove = []
# Perform any method-specific cleanup. This should prevent the need to
# overwrite __call__
self._call_cleanup(input_fp,
output_dir,
params,
job_prefix,
poll_directly,
suppress_submit_jobs)
try:
import h5py
h5py_lib_version = (h5py.__version__ +
' (HDF5 version: %s)' % h5py.version.hdf5_version)
except ImportError:
h5py_lib_version = "Not installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
break
if which('swarm') is None:
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if (opts.suppress_unit_tests and opts.suppress_script_usage_tests):
option_parser.error(
"You're suppressing both test types. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of QIIME's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
qiime_test_data_dir = join(get_qiime_project_dir(), 'qiime_test_data')
qiime_test_data_dir_exists = exists(qiime_test_data_dir)
if not opts.suppress_script_usage_tests and qiime_test_data_dir_exists:
if opts.script_usage_tests is not None:
script_usage_tests = opts.script_usage_tests.split(',')
else:
script_usage_tests = None
# Run the script usage testing functionality
script_usage_result_summary, has_script_usage_example_failures = \
run_script_usage_tests(
test_data_dir=qiime_test_data_dir,
scripts_dir=get_qiime_scripts_dir(),
working_dir=qiime_config['temp_dir'],
verbose=True,
tests=script_usage_tests,
force_overwrite=True,
timeout=240)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % '\n'.join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due " +\
"to missing external applications.\nDepending on the QIIME features " +\
"you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not opts.suppress_script_usage_tests:
if qiime_test_data_dir_exists:
print "\nScript usage test result summary\n--------------------------------\n"
print script_usage_result_summary
else:
print "\nCould not run script usage tests because the directory %s does not exist." % qiime_test_data_dir
print ""
# If script usage tests weren't suppressed, the qiime_test_data dir must
# exist and we can't have any failures.
script_usage_tests_success = (opts.suppress_script_usage_tests or
(qiime_test_data_dir_exists and
not has_script_usage_example_failures))
# If any of the unit tests or script usage tests fail, or if we have any
# missing application errors, use return code 1 (as python's unittest
# module does to indicate one or more failures).
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and
script_usage_tests_success):
return_code = 0
return return_code
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
unittest_glob = opts.unittest_glob
temp_filepath = opts.temp_filepath
script_usage_tests = opts.script_usage_tests
suppress_unit_tests = opts.suppress_unit_tests
suppress_script_usage_tests = opts.suppress_script_usage_tests
# since the test data is in the tests folder just add scripts_test_data
emperor_test_data_dir = join(abspath(dirname(__file__)),
'scripts_test_data/')
# offer the option for the user to pass the scripts dir from the command
# line since there is no other way to get the scripts dir. If not provided
# the base structure of the repository will be assumed. Note that for both
# cases we are using absolute paths, to avoid unwanted failures.
if opts.emperor_scripts_dir == None:
emperor_scripts_dir = abspath(join(get_emperor_project_dir(),
'scripts/'))
else:
emperor_scripts_dir = abspath(opts.emperor_scripts_dir)
# make a sanity check
if (suppress_unit_tests and suppress_script_usage_tests):
option_parser.error("All tests have been suppresed. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of Emperor's unit tests, and keep track of any files which
# fail unit tests, note that these are the unit tests only
if not suppress_unit_tests:
unittest_names = []
if not unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root,name))
else:
for fp in glob(unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
script_usage_failures = 0
# choose to run some of the script usage tests or all the available ones
if not suppress_script_usage_tests and exists(emperor_test_data_dir) and\
exists(emperor_scripts_dir):
if script_usage_tests != None:
script_tests = script_usage_tests.split(',')
else:
script_tests = None
# Run the script usage testing functionality
script_usage_result_summary, script_usage_failures = \
run_script_usage_tests(qiime_test_data_dir=emperor_test_data_dir,
qiime_scripts_dir=emperor_scripts_dir,
working_dir=temp_filepath, verbose=True,
tests=script_tests, failure_log_fp=None, force_overwrite=False)
print "==============\nResult summary\n=============="
if not suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" %'\n'.join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due "+\
"to missing external applications.\nDepending on the Emperor "+\
"features you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not(missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not suppress_script_usage_tests:
if exists(emperor_test_data_dir) and exists(emperor_scripts_dir):
print "\nScript usage test result summary"+\
"\n------------------------------------\n"
print script_usage_result_summary
else:
print ("\nCould not run script usage tests.\nThe Emperor scripts "
"directory could not be automatically located, try supplying "
" it manually using the --emperor_scripts_dir option.")
# In case there were no failures of any type, exit with a return code of 0
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and
script_usage_failures == 0):
return_code = 0
return return_code
def run_script_usage_tests(qiime_test_data_dir,
qiime_scripts_dir,
working_dir,
verbose=False,
tests=None,
failure_log_fp=None,
force_overwrite=False):
""" Test script_usage examples when test data is present in qiime_test_data_dir
These tests are currently used with the qiime_test_data repository, which can
be found at https://github.com/qiime-dev/qiime_test_data
Returns a result summary string and the number of script usage
examples (i.e. commands) that failed.
"""
# process input filepaths and directories
qiime_test_data_dir = abspath(qiime_test_data_dir)
working_dir = join(working_dir, 'script_usage_tests')
if force_overwrite and exists(working_dir):
rmtree(working_dir)
if failure_log_fp != None:
failure_log_fp = abspath(failure_log_fp)
if tests == None:
tests = [
split(d)[1] for d in glob('%s/*' % qiime_test_data_dir) if isdir(d)
]
if verbose:
print 'Tests to run:\n %s' % ' '.join(tests)
addsitedir(qiime_scripts_dir)
failed_tests = []
warnings = []
total_tests = 0
for test in tests:
# import the usage examples - this is possible because we added
# qiime_scripts_dir to the PYTHONPATH above
script_fn = '%s/%s.py' % (qiime_scripts_dir, test)
script = __import__(test)
usage_examples = script.script_info['script_usage']
if verbose:
print 'Testing %d usage examples from: %s' % (len(usage_examples),
script_fn)
# init the test environment
test_input_dir = '%s/%s' % (qiime_test_data_dir, test)
test_working_dir = '%s/%s' % (working_dir, test)
copytree(test_input_dir, test_working_dir)
chdir(test_working_dir)
# remove pre-exisitng output files if any
try:
script_usage_output_to_remove = script.script_info[
'script_usage_output_to_remove']
except KeyError:
script_usage_output_to_remove = []
for e in script_usage_output_to_remove:
rmtree(e.replace('$PWD', getcwd()), ignore_errors=True)
remove_files([e.replace('$PWD', getcwd())], error_on_missing=False)
if verbose:
print ' Running tests in: %s' % getcwd()
print ' Tests:'
for usage_example in usage_examples:
if '%prog' not in usage_example[2]:
warnings.append(
'%s usage examples do not all use %%prog to represent the command name. You may not be running the version of the command that you think you are!'
% test)
cmd = usage_example[2].replace('%prog', script_fn)
if verbose:
print ' %s' % cmd
stdout, stderr, return_value = qiime_system_call(cmd)
total_tests += 1
if return_value != 0:
failed_tests.append((cmd, stdout, stderr, return_value))
if verbose:
print ''
if failure_log_fp:
failure_log_f = open(failure_log_fp, 'w')
if len(failed_tests) == 0:
failure_log_f.write('All script interface tests passed.\n')
else:
i = 0
for cmd, stdout, stderr, return_value in failed_tests:
failure_log_f.write(
'**Failed test %d:\n%s\n\nReturn value: %d\n\nStdout:\n%s\n\nStderr:\n%s\n\n'
% (i, cmd, return_value, stdout, stderr))
failure_log_f.close()
if warnings:
print 'Warnings:'
for warning in warnings:
print ' ' + warning
print ''
result_summary = 'Ran %d commands to test %d scripts. %d of these commands failed.' % (
total_tests, len(tests), len(failed_tests))
if len(failed_tests) > 0:
failed_scripts = set([split(e[0].split()[0])[1] for e in failed_tests])
result_summary += '\nFailed scripts were: %s' % " ".join(
failed_scripts)
if failure_log_fp:
result_summary += "\nFailures are summarized in %s" % failure_log_fp
rmtree(working_dir)
return result_summary, len(failed_tests)
try:
import h5py
h5py_lib_version = (
h5py.__version__ + ' (HDF5 version: %s)' % h5py.version.hdf5_version)
except ImportError:
h5py_lib_version = "Not installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
break
if which('swarm') is None:
def __call__(self,
input_fp,
output_dir,
params,
job_prefix=None,
poll_directly=False,
suppress_submit_jobs=False):
""" """
# Generate a list of files and directories that will be cleaned up
self.files_to_remove = []
# Allow the user to override the default job_prefix (defined by the
# base classes)
if job_prefix is None:
job_prefix = self._get_random_job_prefix(self._job_prefix)
# A temporary output directory is created in output_dir named
# job_prefix. Output files are then moved from the temporary
# directory to the output directory when they are complete,
# allowing a poller to detect when runs complete by the presence
# of their output files.
working_dir = join(output_dir, job_prefix)
try:
makedirs(working_dir)
self.files_to_remove.append(working_dir)
except OSError:
# working dir already exists
pass
# Perform any method-specific setup. This should prevent the need to
# overwrite __call__
self._call_initialization(input_fp,
output_dir,
params,
job_prefix,
poll_directly,
suppress_submit_jobs)
# split the input filepath into directory and filename, base filename and
# extension for use in naming other files
try:
input_dir, input_fn = split(input_fp)
input_file_basename, input_ext = splitext(input_fn)
except AttributeError:
# THIS IS AWFUL - SOME OF THE SCRIPTS PASS A LIST, SO THE
# PREVIOUS BLOCK WON'T WORK... WHAT DO WE WANT TO DO?
input_dir, input_fn = split(input_fp[0])
input_file_basename, input_ext = splitext(input_fn)
# Split the input file into the individual job input files. Add the
# individual job files to the files_to_remove list
input_fps, remove_input_on_completion = self._input_splitter(
input_fp,
params,
self._jobs_to_start,
job_prefix,
working_dir)
if remove_input_on_completion:
self.files_to_remove += input_fps
# Perform any method-specific setup (e.g., formatting a BLAST database)
self._precommand_initiation(input_fp, output_dir, working_dir, params)
# Generate the list of commands to be pushed out to workers
# and the list of output files generated by each job.
commands, job_result_filepaths = self._get_job_commands(input_fps,
output_dir,
params,
job_prefix,
working_dir)
self.files_to_remove += \
self._identify_files_to_remove(job_result_filepaths, params)
# Generate the output clean-up files
merge_map_filepath, deletion_list_filepath, expected_files_filepath =\
self._initialize_output_cleanup_files(job_result_filepaths,
output_dir,
working_dir,
input_file_basename,
params)
# Set up poller apparatus if the user does not suppress polling
if not self._suppress_polling:
poller_command = self._initiate_polling(job_result_filepaths,
working_dir,
poll_directly,
merge_map_filepath,
deletion_list_filepath,
expected_files_filepath)
# If the poller should be run in the same way as the other commands
# (rather than by the current process), add it to the list of commands
if not poll_directly:
commands.append(poller_command)
# Build the filepath for the 'jobs script'. Add that file to the
# files_to_remove list.
jobs_fp = join(working_dir, job_prefix + 'jobs.txt')
self._write_jobs_file(commands, jobs_fp)
self.files_to_remove.append(jobs_fp)
# submit the jobs file using cluster_jobs, if not suppressed by the
# user
if not suppress_submit_jobs:
stdout, stderr, return_value = self._submit_jobs(
jobs_fp=jobs_fp, job_prefix=job_prefix)
# If the poller is going to be run by the current process,
# start polling
if poll_directly:
# IMPORTANT: the following line MUST use qiime_system_call()
# instead of subprocess.call, .check_call, or .check_output in case
# we are invoked in a child process with PIPEs (a deadlock will
# occur otherwise). This can happen if this code is tested by
# all_tests.py, for example.
stdout, stderr, return_value = qiime_system_call(poller_command)
if return_value != 0:
print '**Error occuring when calling the poller directly. ' +\
'Jobs may have been submitted, but are not being polled.'
print stderr
print poller_command
exit(-1)
self.files_to_remove = []
# Perform any method-specific cleanup. This should prevent the need to
# overwrite __call__
self._call_cleanup(input_fp,
output_dir,
params,
job_prefix,
poll_directly,
suppress_submit_jobs)
def submit_illumina_and_split_lib(data_access,fastq_files,metadata_study_id,
input_dir):
"""
Illumina Loading: This function takes the fasta filenames and using
that path, determines the location of the split-library and picked-otu
files. Once file locations have been determined, it moves the files to
the DB machine and load the files into the DB.
"""
# get DB connection and cursor
con = data_access.getSFFDatabaseConnection()
cur = con.cursor()
### this may help in speeding up loading but shouldn't be necessary
#print 'Rebuilding PK_SPLIT_LIBRARY_READ_MAP...'
#cur.execute('alter index "SFF"."PK_SPLIT_LIBRARY_READ_MAP" rebuild ')
#cur = con.cursor()
# check if study exists
study_id_exists=data_access.checkIfStudyIdExists(metadata_study_id)
print "Study ID exists: " + str(study_id_exists)
# get temp filename
alphabet = "ABCDEFGHIJKLMNOPQRSTUZWXYZ"
alphabet += alphabet.lower()
alphabet += "01234567890"
random_fname=''.join([choice(alphabet) for i in range(10)])
tmp_filename ='_'+random_fname+'_'+strftime("%Y_%m_%d_%H_%M_%S")
# get fastq filenames
fastq_filenames=fastq_files.split(',')
seq_run_id=0
analysis_id=0
split_lib_input_checksums=[]
### by disabling constraints you can speed up loading as well, but shouldn't
### be necessary
#valid = data_access.disableTableConstraints()
#print "Disabled table constraints"
#split the fastq filenames and determine filepaths
for fastq_fname in fastq_filenames:
input_fname, input_ext = splitext(split(fastq_fname)[-1])
input_basename, input_ext = splitext(fastq_fname)
# get analysis notes
analysis_notes=split(input_basename)[0]
# get md5 for raw fastq files
fastq_md5 = safe_md5(open(fastq_fname)).hexdigest()
print 'MD5 is: %s' % str(fastq_md5)
# create an analysis row in analysis table
if analysis_id==0:
analysis_id=data_access.createAnalysis(metadata_study_id)
# check if fastq info already loaded
fastq_exists=data_access.checkIfSFFExists(fastq_md5)
print 'fastq in database? %s' % str(fastq_exists)
# if fastq info not loaded, then insert into DB
if not fastq_exists:
if seq_run_id==0:
seq_run_id=data_access.createSequencingRun(True,'ILLUMINA',
None,seq_run_id)
# get sequence count
if fastq_fname.endswith('.gz'):
count_seqs_cmd = "zcat %s | grep ^@ | wc -l" % (fastq_fname)
else:
count_seqs_cmd="grep ^@ %s | wc -l" % (fastq_fname)
o,e,r = qiime_system_call(count_seqs_cmd)
seq_counts = o.strip()
# get header length and # of flows (length of seq)
fastq_fname_open=open(fastq_fname)
first_seq_fastq=get_top_fastq_two_lines(fastq_fname_open)
header_length=len(first_seq_fastq[1])
num_flows=len(first_seq_fastq[1])
# insert fastq info
valid=data_access.addSFFFileInfo(True,input_fname,
seq_counts,
header_length,
None,
num_flows,
None,
None,
None,
fastq_md5,seq_run_id)
else:
seq_run_id=data_access.getSeqRunIDUsingMD5(fastq_md5)
print 'sequence_run_id is: %s' % str(seq_run_id)
# get md5 for split-library input
split_lib_input_md5sum=safe_md5(MD5Wrap(fastq_filenames)).hexdigest()
print split_lib_input_md5sum
print 'Finished loading the processed ILLUMINA data!'
print 'Run ID: %s' % seq_run_id
print 'Analysis ID: %s' % analysis_id
# update analysis table with seq_run_id
valid=data_access.updateAnalysisWithSeqRunID(True,analysis_id,seq_run_id)
if not valid:
raise ValueError, 'Error: Unable to append SEQ_RUN_ID into ANALYSIS table!'
return analysis_id,input_dir,seq_run_id,split_lib_input_md5sum
try:
import h5py
h5py_lib_version = (h5py.__version__ +
' (HDF5 version: %s)' % h5py.version.hdf5_version)
except ImportError:
h5py_lib_version = "Not installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
break
script_info = {}
def make_line_plot(dir_path,
data_file_link,
background_color,
label_color,
xy_coords,
props,
x_len=8,
y_len=4,
draw_axes=False,
generate_eps=True):
""" Write a line plot
xy_coords: a dict of form {series_label:([x data], [y data], point_marker, color)}
(code adapted from Micah Hamady's code)
"""
rc('font', size='8')
rc('axes', linewidth=.5, edgecolor=label_color)
rc('axes', labelsize=8)
rc('xtick', labelsize=8)
rc('ytick', labelsize=8)
fig = figure(figsize=(x_len, y_len))
mtitle = props.get("title", "Groups")
x_label = props.get("xlabel", "X")
y_label = props.get("ylabel", "Y")
title('%s' % mtitle, fontsize='10', color=label_color)
xlabel(x_label, fontsize='8', color=label_color)
ylabel(y_label, fontsize='8', color=label_color)
sorted_keys = sorted(xy_coords.keys())
labels = []
for s_label in sorted_keys:
s_data = xy_coords[s_label]
c = s_data[3]
m = s_data[2]
plot(s_data[0],
s_data[1],
c=c,
marker=m,
label=s_label,
linewidth=.1,
ms=5,
alpha=1.0)
fp = FontProperties()
fp.set_size('8')
legend(prop=fp, loc=0)
show()
img_name = 'scree_plot.png'
savefig(os.path.join(dir_path, img_name),
dpi=80,
facecolor=background_color)
# Create zipped eps files
eps_link = ""
if generate_eps:
eps_img_name = str('scree_plot.eps')
savefig(os.path.join(dir_path, eps_img_name), format='eps')
out, err, retcode = qiime_system_call(
"gzip -f " + os.path.join(dir_path, eps_img_name))
eps_link = DOWNLOAD_LINK % (
(os.path.join(data_file_link, eps_img_name) + ".gz"),
"Download Figure")
return os.path.join(data_file_link, img_name), eps_link
def create_personal_results(mapping_fp,
distance_matrix_fp,
collated_dir_fp,
output_fp, prefs_fp,
personal_id_field,
otu_table,
parameter_fp,
personal_ids=None,
column_title='Self',
individual_titles=None,
category_to_split='BodySite',
time_series_category='WeeksSinceStart',
suppress_alpha_rarefaction=False,
verbose=False):
map_as_list, header, comments = parse_mapping_file(open(mapping_fp, 'U'))
try:
personal_id_index = header.index(personal_id_field)
except ValueError:
raise ValueError("personal id field (%s) is not a mapping file column header" %
personal_id_field)
header.append(column_title)
if personal_ids == None:
personal_ids = get_personal_ids(map_as_list, personal_id_index)
else:
for id in personal_ids.split(','):
if id not in get_personal_ids(map_as_list, personal_id_index):
raise ValueError('%s is not an id in the mapping file.' %id)
personal_ids = personal_ids.split(',')
output_directories = []
makedirs(output_fp)
otu_table_title = otu_table.split('/')[-1].split('.')
for person_of_interest in personal_ids:
makedirs(join(output_fp, person_of_interest))
pcoa_dir = join(output_fp, person_of_interest, "beta_diversity")
rarefaction_dir = join(output_fp, person_of_interest, "alpha_rarefaction")
area_plots_dir = join(output_fp, person_of_interest, "time_series")
output_directories.append(pcoa_dir)
output_directories.append(rarefaction_dir)
output_directories.append(area_plots_dir)
personal_mapping_file_fp = join(output_fp, person_of_interest, "mapping_file.txt")
html_fp = join(output_fp, person_of_interest, "index.html")
personal_map = create_personal_mapping_file(map_as_list,
header,
comments,
person_of_interest,
personal_mapping_file_fp,
personal_id_index,
individual_titles)
create_index_html(person_of_interest, html_fp)
column_title_index = header.index(column_title)
column_title_values = set([e[column_title_index] for e in personal_map])
cat_index = header.index(category_to_split)
cat_values = set([e[cat_index] for e in personal_map])
## Alpha rarefaction steps
if not suppress_alpha_rarefaction:
cmd = "make_rarefaction_plots.py -i %s -m %s -p %s -o %s" % (collated_dir_fp,
personal_mapping_file_fp,
prefs_fp,
rarefaction_dir)
if verbose:
print cmd
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
## Beta diversity steps
cmd = "make_3d_plots.py -m %s -p %s -i %s -o %s" % (personal_mapping_file_fp,
prefs_fp,
distance_matrix_fp,
pcoa_dir)
if verbose:
print cmd
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
## Split OTU table into self/other per-body-site tables
cmd = "split_otu_table.py -i %s -m %s -f %s -o %s" % (otu_table,
personal_mapping_file_fp,
column_title,
area_plots_dir)
if verbose:
print cmd
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
for column_title_value in column_title_values:
print column_title_value
biom_fp = join(area_plots_dir, '%s_%s.%s' % (otu_table_title[0], column_title_value, otu_table_title[1]))
body_site_dir = join(area_plots_dir, column_title_value)
cmd = "split_otu_table.py -i %s -m %s -f %s -o %s" % (biom_fp,
personal_mapping_file_fp,
category_to_split,
body_site_dir)
if verbose:
print cmd
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
for cat_value in cat_values:
otu_table_fp = join(body_site_dir, "otu_table_%s_%s.biom" % (column_title_value, cat_value))
print otu_table_fp
if exists(otu_table_fp):
# Not supporting parameter files yet
#if parameter_fp == None:
# parameter_fp = ''
#else:
# parameter_fp = '-p %s' %parameter_fp
plots = join(area_plots_dir, "taxa_plots_%s_%s" % (column_title_value, cat_value))
cmd = "summarize_taxa_through_plots.py -i %s -o %s -c %s -m %s -s" % (otu_table_fp,
plots,
time_series_category,
personal_mapping_file_fp)
#parameter_fp)
if verbose:
print cmd
stdout, stderr, return_code = qiime_system_call(cmd)
if return_code != 0:
raise ValueError("Command failed!\nCommand: %s\n Stdout: %s\n Stderr: %s\n" %\
(cmd, stdout, stderr))
return output_directories
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if (opts.suppress_unit_tests and \
opts.suppress_script_tests and \
opts.suppress_script_usage_tests):
option_parser.error(
"You're suppressing all three test types. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of QIIME's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
bad_scripts = []
if not opts.suppress_script_tests:
# Run through all of QIIME's scripts, and pass -h to each one. If the
# resulting stdout does not being with the Usage text, that is an
# indicator of something being wrong with the script. Issues that would
# cause that are bad import statements in the script, SyntaxErrors, or
# other failures prior to running qiime.util.parse_command_line_parameters.
try:
scripts_dir = get_qiime_scripts_dir()
script_directory_found = True
except AssertionError:
script_directory_found = False
if script_directory_found:
script_names = []
script_names = glob('%s/*py' % scripts_dir)
script_names.sort()
for script_name in script_names:
script_good_pattern = re.compile('^Usage: %s' %
split(script_name)[1])
print "Testing %s." % script_name
command = '%s %s -h' % (python_name, script_name)
stdout, stderr, return_value = qiime_system_call(command)
if not script_good_pattern.search(stdout):
bad_scripts.append(script_name)
num_script_usage_example_failures = 0
qiime_test_data_dir = qiime_config['qiime_test_data_dir']
if not opts.suppress_script_usage_tests and qiime_test_data_dir != None:
# Run the script usage testing functionality
script_usage_result_summary, num_script_usage_example_failures = \
run_script_usage_tests(
qiime_test_data_dir=qiime_test_data_dir,
qiime_scripts_dir=qiime_config['qiime_scripts_dir'],
working_dir=qiime_config['temp_dir'],
verbose=True,
tests=None, # runs all
failure_log_fp=None,
force_overwrite=True)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % '\n'.join(
bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due "+\
"to missing external applications.\nDepending on the QIIME features "+\
"you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not opts.suppress_script_tests:
print "\nBasic script test result summary\n--------------------------------\n"
if not script_directory_found:
print "Critical error: Failed to test scripts because the script directory could not be found.\n The most likely explanation for this failure is that you've installed QIIME using setup.py, and forgot to specify the qiime_scripts_dir in your qiime_config file. This value shoud be set either to the directory you provided for --install-scripts, or /usr/local/bin if no value was provided to --install-scripts."
else:
if bad_scripts:
print "Failed the following basic script tests.\n%s" % '\n'.join(
bad_scripts)
else:
print "All basic script tests passed successfully.\n"
qiime_test_data_dir_exists = True
if not opts.suppress_script_usage_tests:
if qiime_test_data_dir:
print "\nScript usage test result summary\n------------------------------------\n"
print script_usage_result_summary
else:
print "\nCould not run script usage tests because qiime_test_data_dir is not defined in your qiime_config."
qiime_test_data_dir_exists = False
print ""
# If any of the unit tests, script tests, or script usage tests fail, or if
# we have any missing application errors or a missing QIIME test data dir
# if script usage tests weren't suppressed, use return code 1 (as python's
# unittest module does to indicate one or more failures).
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0
and len(bad_scripts) == 0
and num_script_usage_example_failures == 0
and qiime_test_data_dir_exists):
return_code = 0
return return_code