def test_parse_args(self):
"""CommandParser.parse_args works for simple cases using optparse
"""
# Skip the test if optparse not available
if not OPTPARSE_AVAILABLE:
raise unittest.SkipTest("'optparse' not available")
p = CommandParser(subparser=OptionParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
slow_cmd.add_option('-a', action='store', dest='a_value')
fast_cmd.add_option('-b', action='store', dest='b_value')
cmd, options, args = p.parse_args(['slow', '-a', 'unspeedy', 'input'])
self.assertEqual(cmd, 'slow')
self.assertEqual(options.a_value, 'unspeedy')
self.assertEqual(args, ['input'])
try:
options.b_value
self.fail(
"Accessing 'b_value' for 'slow' command didn't raise AttributeError"
)
except AttributeError:
pass
cmd, options, args = p.parse_args(['fast', '-b', 'zippy', 'input2'])
self.assertEqual(cmd, 'fast')
self.assertEqual(options.b_value, 'zippy')
self.assertEqual(args, ['input2'])
try:
options.a_value
self.fail(
"Accessing 'a_value' for 'fast' command didn't raise AttributeError"
)
except AttributeError:
pass
def test_add_command(self):
"""CommandParser.add_command works for a single command using argparse
"""
p = CommandParser(subparser=ArgumentParser)
cmd = p.add_command('slow')
self.assertTrue(isinstance(cmd, ArgumentParser))
self.assertEqual(p.list_commands(), ['slow'])
def test_parse_args(self):
"""CommandParser.parse_args works for simple cases using argparse
"""
p = CommandParser(subparser=ArgumentParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
slow_cmd.add_argument('-a', action='store', dest='a_value')
slow_cmd.add_argument('name')
fast_cmd.add_argument('-b', action='store', dest='b_value')
fast_cmd.add_argument('name')
cmd, args = p.parse_args(['slow', '-a', 'unspeedy', 'input'])
self.assertEqual(cmd, 'slow')
self.assertEqual(args.a_value, 'unspeedy')
self.assertEqual(args.name, 'input')
try:
args.b_value
self.fail(
"Accessing 'b_value' for 'slow' command didn't raise AttributeError"
)
except AttributeError:
pass
cmd, args = p.parse_args(['fast', '-b', 'zippy', 'input2'])
self.assertEqual(cmd, 'fast')
self.assertEqual(args.b_value, 'zippy')
self.assertEqual(args.name, 'input2')
try:
args.a_value
self.fail(
"Accessing 'a_value' for 'fast' command didn't raise AttributeError"
)
except AttributeError:
pass
def test_handles_version(self):
"""CommandParser handles version using optparse
"""
# Skip the test if optparse not available
if not OPTPARSE_AVAILABLE:
raise unittest.SkipTest("'optparse' not available")
p = CommandParser(subparser=OptionParser, version="0.1")
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
def test_add_command(self):
"""CommandParser.add_command works for a single command using optparse
"""
# Skip the test if optparse not available
if not OPTPARSE_AVAILABLE:
raise unittest.SkipTest("'optparse' not available")
p = CommandParser(subparser=OptionParser)
cmd = p.add_command('slow')
self.assertTrue(isinstance(cmd, OptionParser))
self.assertEqual(p.list_commands(), ['slow'])
def test_add_multiple_commands(self):
"""CommandParser.add_command works for multiple commands using argparse
"""
p = CommandParser(subparser=ArgumentParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
medium_cmd = p.add_command('medium')
self.assertTrue(isinstance(slow_cmd, ArgumentParser))
self.assertTrue(isinstance(fast_cmd, ArgumentParser))
self.assertTrue(isinstance(medium_cmd, ArgumentParser))
self.assertEqual(p.list_commands(), ['slow', 'fast', 'medium'])
def test_add_multiple_commands(self):
"""CommandParser.add_command works for multiple commands using optparse
"""
# Skip the test if optparse not available
if not OPTPARSE_AVAILABLE:
raise unittest.SkipTest("'optparse' not available")
p = CommandParser(subparser=OptionParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
medium_cmd = p.add_command('medium')
self.assertTrue(isinstance(slow_cmd, OptionParser))
self.assertTrue(isinstance(fast_cmd, OptionParser))
self.assertTrue(isinstance(medium_cmd, OptionParser))
self.assertEqual(p.list_commands(), ['slow', 'fast', 'medium'])
def test_parser_for(self):
"""CommandParser.parser_for returns the correct ArgumentParser
"""
p = CommandParser(subparser=ArgumentParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
medium_cmd = p.add_command('medium')
self.assertEqual(p.parser_for('slow'), slow_cmd)
self.assertEqual(p.parser_for('fast'), fast_cmd)
self.assertEqual(p.parser_for('medium'), medium_cmd)
def test_parser_for(self):
"""CommandParser.parser_for returns the correct OptionParser
"""
# Skip the test if optparse not available
if not OPTPARSE_AVAILABLE:
raise unittest.SkipTest("'optparse' not available")
p = CommandParser(subparser=OptionParser)
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
medium_cmd = p.add_command('medium')
self.assertEqual(p.parser_for('slow'), slow_cmd)
self.assertEqual(p.parser_for('fast'), fast_cmd)
self.assertEqual(p.parser_for('medium'), medium_cmd)
def test_handles_version(self):
"""CommandParser handles version using argparse
"""
p = CommandParser(subparser=ArgumentParser, version="0.1")
slow_cmd = p.add_command('slow')
fast_cmd = p.add_command('fast')
# Remote copy
try:
scp = applications.general.scp(user,host,f,dest)
print "Running %s" % scp
scp.run_subprocess()
except Exception, ex:
raise Exception("Failed to copy %s to %s: %s" % (f,dirn,ex))
#######################################################################
# Main program
#######################################################################
if __name__ == "__main__":
# Set up command line parser
p = CommandParser(description="Utility for managing processed and analysed Illumina "
"sequence data in ANALYSIS_DIR",
version="%prog "+get_version())
# Add info command
p.add_command('info',help="Get information about ANALYSIS_DIR",
usage="%prog info [OPTIONS] ANALYSIS_DIR",
description="Report information on processed Illumina "
"sequence data in ANALYSIS_DIR.")
add_debug_option(p.parser_for('info'))
# Add copy command
p.add_command('copy',help="Copy fastqs from ANALYSIS_DIR",
usage="%prog copy [OPTIONS] ANALYSIS_DIR DEST_DIR",
description="Copy fastqs from ANALYSIS_DIR to DEST_DIR.")
p.parser_for('copy').add_option('--projects',action='store',dest='projects',default=None,
help="Restrict copying to projects matching the "
"supplied pattern")
p.parser_for('copy').add_option('--fastq-dir',action='store',dest='fastq_dir',default=None,
add_runner_option(p)
add_debug_option(p)
def set_debug(debug_flag):
"""Turn on debug output
"""
if debug_flag: logging.getLogger().setLevel(logging.DEBUG)
#######################################################################
# Main program
#######################################################################
if __name__ == "__main__":
# Set up the command line parser
p = CommandParser(description="Automated processing & QC for Illumina sequence data.",
version="%prog "+__version__)
# Add commands
add_config_command(p)
add_setup_command(p)
add_params_command(p)
add_metadata_command(p)
add_samplesheet_command(p)
add_make_fastqs_command(p)
add_setup_analysis_dirs_command(p)
add_run_qc_command(p)
add_publish_qc_command(p)
add_archive_command(p)
add_report_command(p)
add_readme_command(p)
add_clone_command(p)
add_analyse_barcodes_command(p)
def main(args=None):
# Set up the command line parser
p = CommandParser(description="Utility for archiving and curating "
"NGS sequence data.",
version="%prog " + __version__)
# Add commands
#
# Info
p.add_command('info',
help="Get information about a data dir",
usage='%prog info DIR',
description="Print information about DIR and its "
"contents.")
#
# Stage data
p.add_command('stage',
help="Make a staging copy of data",
usage='%prog stage DIR STAGING_DIR',
description="Copy DIR to STAGING_DIR and set up for "
"archiving and curation.")
#
# Initialise a cache subdirectory
p.add_command('init_cache',
help="Initialise a cache subdirectory",
usage='%prog init_cache DIR',
description="Create a cache subdirectory under DIR "
"(if one doesn't already exist) and use this to store "
"information such as MD5 sums for quick lookup.")
#
# List files
p.add_command('list_files',
help="List files filtered by various criteria",
usage='%prog list_files OPTIONS DIR',
description="List files under DIR filtered by criteria "
"specified by one or more OPTIONS.")
p.parser_for('list_files').add_option('--extensions',
action='store',
dest='extensions',
default=None,
help="List files with matching "
"extensions")
p.parser_for('list_files').add_option('--compression',
action='store',
dest='compression',
default=None,
help="List files with matching "
"compression extensions")
p.parser_for('list_files').add_option('--owners',
action='store',
dest='owners',
default=None,
help="List files owned by "
"specified users")
p.parser_for('list_files').add_option('--groups',
action='store',
dest='groups',
default=None,
help="List files assigned to "
"specified groups")
p.parser_for('list_files').add_option('--subdir',
action='store',
dest='subdir',
default=None,
help="List files in "
"subdirectory SUBDIR under "
"DIR")
p.parser_for('list_files').add_option('--sort',
action='store',
dest='sortkeys',
default=None,
help="List files sorted in "
"order according to one or "
"more SORTKEYS ('size',...)")
p.parser_for('list_files').add_option('--minsize',
action='store',
dest='min_size',
default=None,
help="Only report files with "
"size greater than MIN_SIZE")
#
# List primary data
p.add_command('primary_data',
help="List primary data files",
usage='%prog primary_data DIR',
description="List the primary data files found in DIR.")
#
# List primary data (SOLiD)
p.add_command(
'report_solid',
help="List primary data files for SOLiD",
usage='%prog report_solid DIR',
description="List the SOLiD primary data files found in DIR.")
#
# Match primary data to links from analysis dir (SOLiD)
p.add_command('match_solid',
help="Find SOLiD datasets linked from analysis dir",
usage='%prog match_solid DIR ANALYSIS_DIR',
description="Determine which SOLiD datasets found in DIR "
"are also linked from ANALYSIS_DIR.")
#
# List symlinks
p.add_command('symlinks',
help="List symlinks",
usage='%prog symlinks DIR',
description="List the symbolic links found in DIR.")
#
# Md5sums
p.add_command('md5sums',
help="Generate MD5 checksums",
usage='%prog md5sums DIR',
description="Generate MD5 checksums for all files "
"in DIR. Symlinks are not followed.")
p.parser_for('md5sums').add_option(
'-o',
action='store',
dest='outfile',
default=None,
help="Write MD5 sums to OUTFILE (otherwise "
"writes to stdout)")
#
# Find duplicates
p.add_command('duplicates',
help="Find duplicated files",
usage='%prog duplicates DIR [DIR ...]',
description="Look for duplicated files across one or "
"more data directories")
#
# Find duplicates
p.add_command('temp_files',
help="Find temporary files & directories",
usage='%prog temp_files DIR [DIR ...]',
description="Look for temporary files and directories "
"in DIR.")
#
# Look for related directories
p.add_command('related',
help="Locate related data directories",
usage='%prog related DIR SEARCH_DIR [SEARCH_DIR ...]',
description="Look for related directories under one "
"or more search directories.")
#
# Set permissions
p.add_command('set_permissions',
help="Set permissions and ownership",
usage='%prog set_permissions OPTIONS DIR',
description="Set the permissions and ownership of DIR "
"according to the supplied options.")
p.parser_for('set_permissions').add_option('--chmod',
action='store',
dest='mode',
default=None,
help="Set file permissions on "
"files to those specified by "
"MODE")
p.parser_for('set_permissions').add_option('--group',
action='store',
dest='group',
default=None,
help="Set group ownership on "
"files to GROUP")
#
# Compress files
p.add_command('compress',
help="Compress data files",
usage='%prog compress DIR EXT [EXT..]',
description="Compress data files in DIR with matching "
"file extensions using bzip2.")
p.parser_for('compress').add_option('--dry-run',
action='store_true',
dest='dry_run',
default=False,
help="Report actions but don't "
"perform them")
#
# Interactive shell
p.add_command('shell',
help="Run interactively",
usage='%prog shell DIR',
description="Run commands interactively on DIR")
# Process command line
cmd, options, args = p.parse_args()
# Report name and version
print "%s version %s" % (os.path.basename(sys.argv[0]), __version__)
if cmd == 'info':
if len(args) != 1:
sys.stderr.write("Need to supply a data dir\n")
sys.exit(1)
DataDir(args[0]).info()
elif cmd == 'stage':
if len(args) != 2:
sys.stderr.write(
"Need to supply a data dir and staging location\n")
sys.exit(1)
stage_data(args[0], args[1])
elif cmd == 'init_cache':
DataDir(args[0]).init_cache()
elif cmd == 'list_files':
list_files(args[0],
extensions=(None if options.extensions is None \
else options.extensions.split(',')),
owners=(None if options.owners is None \
else options.owners.split(',')),
groups=(None if options.groups is None \
else options.groups.split(',')),
compression=(None if options.compression is None \
else options.compression.split(',')),
subdir=options.subdir,
sort_keys=(None if options.sortkeys is None \
else options.sortkeys.split(',')),
min_size=options.min_size)
elif cmd == 'primary_data':
find_primary_data(args[0])
elif cmd == 'report_solid':
SolidDataDir(args[0]).report()
elif cmd == 'match_solid':
if len(args) < 2:
sys.stderr.write("Need to supply a SOLiD data dir and at "
"least one analysis directory\n")
sys.exit(1)
SolidDataDir(args[0]).match_primary_data(*args[1:])
elif cmd == 'symlinks':
find_symlinks(args[0])
elif cmd == 'md5sums':
find_md5sums(args[0], options.outfile)
elif cmd == 'duplicates':
find_duplicates(*args)
elif cmd == 'temp_files':
find_tmp_files(args[0])
elif cmd == 'set_permissions':
DataDir(args[0]).set_permissions(mode=options.mode,
group=options.group)
elif cmd == 'compress':
if len(args) < 2:
sys.stderr.write("Need to supply a data dir and at least "
"one extension\n")
sys.exit(1)
compress_files(args[0], args[1:], dry_run=options.dry_run)
elif cmd == 'related':
find_related(args[0])
elif cmd == 'shell':
Shell(args[0]).cmdloop()