Ejemplo n.º 1
0
 def test_parse_args(self):
     """CommandParser.parse_args works for simple cases using optparse
     """
     # Skip the test if optparse not available
     if not OPTPARSE_AVAILABLE:
         raise unittest.SkipTest("'optparse' not available")
     p = CommandParser(subparser=OptionParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     slow_cmd.add_option('-a', action='store', dest='a_value')
     fast_cmd.add_option('-b', action='store', dest='b_value')
     cmd, options, args = p.parse_args(['slow', '-a', 'unspeedy', 'input'])
     self.assertEqual(cmd, 'slow')
     self.assertEqual(options.a_value, 'unspeedy')
     self.assertEqual(args, ['input'])
     try:
         options.b_value
         self.fail(
             "Accessing 'b_value' for 'slow' command didn't raise AttributeError"
         )
     except AttributeError:
         pass
     cmd, options, args = p.parse_args(['fast', '-b', 'zippy', 'input2'])
     self.assertEqual(cmd, 'fast')
     self.assertEqual(options.b_value, 'zippy')
     self.assertEqual(args, ['input2'])
     try:
         options.a_value
         self.fail(
             "Accessing 'a_value' for 'fast' command didn't raise AttributeError"
         )
     except AttributeError:
         pass
Ejemplo n.º 2
0
 def test_add_command(self):
     """CommandParser.add_command works for a single command using argparse
     """
     p = CommandParser(subparser=ArgumentParser)
     cmd = p.add_command('slow')
     self.assertTrue(isinstance(cmd, ArgumentParser))
     self.assertEqual(p.list_commands(), ['slow'])
Ejemplo n.º 3
0
 def test_parse_args(self):
     """CommandParser.parse_args works for simple cases using argparse
     """
     p = CommandParser(subparser=ArgumentParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     slow_cmd.add_argument('-a', action='store', dest='a_value')
     slow_cmd.add_argument('name')
     fast_cmd.add_argument('-b', action='store', dest='b_value')
     fast_cmd.add_argument('name')
     cmd, args = p.parse_args(['slow', '-a', 'unspeedy', 'input'])
     self.assertEqual(cmd, 'slow')
     self.assertEqual(args.a_value, 'unspeedy')
     self.assertEqual(args.name, 'input')
     try:
         args.b_value
         self.fail(
             "Accessing 'b_value' for 'slow' command didn't raise AttributeError"
         )
     except AttributeError:
         pass
     cmd, args = p.parse_args(['fast', '-b', 'zippy', 'input2'])
     self.assertEqual(cmd, 'fast')
     self.assertEqual(args.b_value, 'zippy')
     self.assertEqual(args.name, 'input2')
     try:
         args.a_value
         self.fail(
             "Accessing 'a_value' for 'fast' command didn't raise AttributeError"
         )
     except AttributeError:
         pass
Ejemplo n.º 4
0
 def test_handles_version(self):
     """CommandParser handles version using optparse
     """
     # Skip the test if optparse not available
     if not OPTPARSE_AVAILABLE:
         raise unittest.SkipTest("'optparse' not available")
     p = CommandParser(subparser=OptionParser, version="0.1")
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
Ejemplo n.º 5
0
 def test_add_command(self):
     """CommandParser.add_command works for a single command using optparse
     """
     # Skip the test if optparse not available
     if not OPTPARSE_AVAILABLE:
         raise unittest.SkipTest("'optparse' not available")
     p = CommandParser(subparser=OptionParser)
     cmd = p.add_command('slow')
     self.assertTrue(isinstance(cmd, OptionParser))
     self.assertEqual(p.list_commands(), ['slow'])
Ejemplo n.º 6
0
 def test_add_multiple_commands(self):
     """CommandParser.add_command works for multiple commands using argparse
     """
     p = CommandParser(subparser=ArgumentParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     medium_cmd = p.add_command('medium')
     self.assertTrue(isinstance(slow_cmd, ArgumentParser))
     self.assertTrue(isinstance(fast_cmd, ArgumentParser))
     self.assertTrue(isinstance(medium_cmd, ArgumentParser))
     self.assertEqual(p.list_commands(), ['slow', 'fast', 'medium'])
Ejemplo n.º 7
0
 def test_add_multiple_commands(self):
     """CommandParser.add_command works for multiple commands using optparse
     """
     # Skip the test if optparse not available
     if not OPTPARSE_AVAILABLE:
         raise unittest.SkipTest("'optparse' not available")
     p = CommandParser(subparser=OptionParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     medium_cmd = p.add_command('medium')
     self.assertTrue(isinstance(slow_cmd, OptionParser))
     self.assertTrue(isinstance(fast_cmd, OptionParser))
     self.assertTrue(isinstance(medium_cmd, OptionParser))
     self.assertEqual(p.list_commands(), ['slow', 'fast', 'medium'])
Ejemplo n.º 8
0
 def test_parser_for(self):
     """CommandParser.parser_for returns the correct ArgumentParser
     """
     p = CommandParser(subparser=ArgumentParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     medium_cmd = p.add_command('medium')
     self.assertEqual(p.parser_for('slow'), slow_cmd)
     self.assertEqual(p.parser_for('fast'), fast_cmd)
     self.assertEqual(p.parser_for('medium'), medium_cmd)
Ejemplo n.º 9
0
 def test_parser_for(self):
     """CommandParser.parser_for returns the correct OptionParser
     """
     # Skip the test if optparse not available
     if not OPTPARSE_AVAILABLE:
         raise unittest.SkipTest("'optparse' not available")
     p = CommandParser(subparser=OptionParser)
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
     medium_cmd = p.add_command('medium')
     self.assertEqual(p.parser_for('slow'), slow_cmd)
     self.assertEqual(p.parser_for('fast'), fast_cmd)
     self.assertEqual(p.parser_for('medium'), medium_cmd)
Ejemplo n.º 10
0
 def test_handles_version(self):
     """CommandParser handles version using argparse
     """
     p = CommandParser(subparser=ArgumentParser, version="0.1")
     slow_cmd = p.add_command('slow')
     fast_cmd = p.add_command('fast')
Ejemplo n.º 11
0
        # Remote copy
        try:
            scp = applications.general.scp(user,host,f,dest)
            print "Running %s" % scp
            scp.run_subprocess()
        except Exception, ex:
            raise Exception("Failed to copy %s to %s: %s" % (f,dirn,ex))

#######################################################################
# Main program
#######################################################################

if __name__ == "__main__":
    # Set up command line parser
    p = CommandParser(description="Utility for managing processed and analysed Illumina "
                      "sequence data in ANALYSIS_DIR",
                      version="%prog "+get_version())
    # Add info command
    p.add_command('info',help="Get information about ANALYSIS_DIR",
                  usage="%prog info [OPTIONS] ANALYSIS_DIR",
                  description="Report information on processed Illumina "
                  "sequence data in ANALYSIS_DIR.")
    add_debug_option(p.parser_for('info'))
    # Add copy command
    p.add_command('copy',help="Copy fastqs from ANALYSIS_DIR",
                  usage="%prog copy [OPTIONS] ANALYSIS_DIR DEST_DIR",
                  description="Copy fastqs from ANALYSIS_DIR to DEST_DIR.")
    p.parser_for('copy').add_option('--projects',action='store',dest='projects',default=None,
                                    help="Restrict copying to projects matching the "
                                    "supplied pattern")
    p.parser_for('copy').add_option('--fastq-dir',action='store',dest='fastq_dir',default=None,
    add_runner_option(p)
    add_debug_option(p)

def set_debug(debug_flag):
    """Turn on debug output
    """
    if debug_flag: logging.getLogger().setLevel(logging.DEBUG)

#######################################################################
# Main program
#######################################################################

if __name__ == "__main__":

    # Set up the command line parser
    p = CommandParser(description="Automated processing & QC for Illumina sequence data.",
                      version="%prog "+__version__)
    # Add commands
    add_config_command(p)
    add_setup_command(p)
    add_params_command(p)
    add_metadata_command(p)
    add_samplesheet_command(p)
    add_make_fastqs_command(p)
    add_setup_analysis_dirs_command(p)
    add_run_qc_command(p)
    add_publish_qc_command(p)
    add_archive_command(p)
    add_report_command(p)
    add_readme_command(p)
    add_clone_command(p)
    add_analyse_barcodes_command(p)
Ejemplo n.º 13
0
def main(args=None):

    # Set up the command line parser
    p = CommandParser(description="Utility for archiving and curating "
                      "NGS sequence data.",
                      version="%prog " + __version__)
    # Add commands
    #
    # Info
    p.add_command('info',
                  help="Get information about a data dir",
                  usage='%prog info DIR',
                  description="Print information about DIR and its "
                  "contents.")
    #
    # Stage data
    p.add_command('stage',
                  help="Make a staging copy of data",
                  usage='%prog stage DIR STAGING_DIR',
                  description="Copy DIR to STAGING_DIR and set up for "
                  "archiving and curation.")
    #
    # Initialise a cache subdirectory
    p.add_command('init_cache',
                  help="Initialise a cache subdirectory",
                  usage='%prog init_cache DIR',
                  description="Create a cache subdirectory under DIR "
                  "(if one doesn't already exist) and use this to store "
                  "information such as MD5 sums for quick lookup.")
    #
    # List files
    p.add_command('list_files',
                  help="List files filtered by various criteria",
                  usage='%prog list_files OPTIONS DIR',
                  description="List files under DIR filtered by criteria "
                  "specified by one or more OPTIONS.")
    p.parser_for('list_files').add_option('--extensions',
                                          action='store',
                                          dest='extensions',
                                          default=None,
                                          help="List files with matching "
                                          "extensions")
    p.parser_for('list_files').add_option('--compression',
                                          action='store',
                                          dest='compression',
                                          default=None,
                                          help="List files with matching "
                                          "compression extensions")
    p.parser_for('list_files').add_option('--owners',
                                          action='store',
                                          dest='owners',
                                          default=None,
                                          help="List files owned by "
                                          "specified users")
    p.parser_for('list_files').add_option('--groups',
                                          action='store',
                                          dest='groups',
                                          default=None,
                                          help="List files assigned to "
                                          "specified groups")
    p.parser_for('list_files').add_option('--subdir',
                                          action='store',
                                          dest='subdir',
                                          default=None,
                                          help="List files in "
                                          "subdirectory SUBDIR under "
                                          "DIR")
    p.parser_for('list_files').add_option('--sort',
                                          action='store',
                                          dest='sortkeys',
                                          default=None,
                                          help="List files sorted in "
                                          "order according to one or "
                                          "more SORTKEYS ('size',...)")
    p.parser_for('list_files').add_option('--minsize',
                                          action='store',
                                          dest='min_size',
                                          default=None,
                                          help="Only report files with "
                                          "size greater than MIN_SIZE")
    #
    # List primary data
    p.add_command('primary_data',
                  help="List primary data files",
                  usage='%prog primary_data DIR',
                  description="List the primary data files found in DIR.")
    #
    # List primary data (SOLiD)
    p.add_command(
        'report_solid',
        help="List primary data files for SOLiD",
        usage='%prog report_solid DIR',
        description="List the SOLiD primary data files found in DIR.")
    #
    # Match primary data to links from analysis dir (SOLiD)
    p.add_command('match_solid',
                  help="Find SOLiD datasets linked from analysis dir",
                  usage='%prog match_solid DIR ANALYSIS_DIR',
                  description="Determine which SOLiD datasets found in DIR "
                  "are also linked from ANALYSIS_DIR.")
    #
    # List symlinks
    p.add_command('symlinks',
                  help="List symlinks",
                  usage='%prog symlinks DIR',
                  description="List the symbolic links found in DIR.")
    #
    # Md5sums
    p.add_command('md5sums',
                  help="Generate MD5 checksums",
                  usage='%prog md5sums DIR',
                  description="Generate MD5 checksums for all files "
                  "in DIR. Symlinks are not followed.")
    p.parser_for('md5sums').add_option(
        '-o',
        action='store',
        dest='outfile',
        default=None,
        help="Write MD5 sums to OUTFILE (otherwise "
        "writes to stdout)")
    #
    # Find duplicates
    p.add_command('duplicates',
                  help="Find duplicated files",
                  usage='%prog duplicates DIR [DIR ...]',
                  description="Look for duplicated files across one or "
                  "more data directories")
    #
    # Find duplicates
    p.add_command('temp_files',
                  help="Find temporary files & directories",
                  usage='%prog temp_files DIR [DIR ...]',
                  description="Look for temporary files and directories "
                  "in DIR.")
    #
    # Look for related directories
    p.add_command('related',
                  help="Locate related data directories",
                  usage='%prog related DIR SEARCH_DIR [SEARCH_DIR ...]',
                  description="Look for related directories under one "
                  "or more search directories.")
    #
    # Set permissions
    p.add_command('set_permissions',
                  help="Set permissions and ownership",
                  usage='%prog set_permissions OPTIONS DIR',
                  description="Set the permissions and ownership of DIR "
                  "according to the supplied options.")
    p.parser_for('set_permissions').add_option('--chmod',
                                               action='store',
                                               dest='mode',
                                               default=None,
                                               help="Set file permissions on "
                                               "files to those specified by "
                                               "MODE")
    p.parser_for('set_permissions').add_option('--group',
                                               action='store',
                                               dest='group',
                                               default=None,
                                               help="Set group ownership on "
                                               "files to GROUP")
    #
    # Compress files
    p.add_command('compress',
                  help="Compress data files",
                  usage='%prog compress DIR EXT [EXT..]',
                  description="Compress data files in DIR with matching "
                  "file extensions using bzip2.")
    p.parser_for('compress').add_option('--dry-run',
                                        action='store_true',
                                        dest='dry_run',
                                        default=False,
                                        help="Report actions but don't "
                                        "perform them")
    #
    # Interactive shell
    p.add_command('shell',
                  help="Run interactively",
                  usage='%prog shell DIR',
                  description="Run commands interactively on DIR")
    # Process command line
    cmd, options, args = p.parse_args()

    # Report name and version
    print "%s version %s" % (os.path.basename(sys.argv[0]), __version__)

    if cmd == 'info':
        if len(args) != 1:
            sys.stderr.write("Need to supply a data dir\n")
            sys.exit(1)
        DataDir(args[0]).info()
    elif cmd == 'stage':
        if len(args) != 2:
            sys.stderr.write(
                "Need to supply a data dir and staging location\n")
            sys.exit(1)
        stage_data(args[0], args[1])
    elif cmd == 'init_cache':
        DataDir(args[0]).init_cache()
    elif cmd == 'list_files':
        list_files(args[0],
                   extensions=(None if options.extensions is None \
                               else options.extensions.split(',')),
                   owners=(None if options.owners is None \
                           else options.owners.split(',')),
                   groups=(None if options.groups is None \
                           else options.groups.split(',')),
                   compression=(None if options.compression is None \
                                else options.compression.split(',')),
                   subdir=options.subdir,
                   sort_keys=(None if options.sortkeys is None \
                              else options.sortkeys.split(',')),
                   min_size=options.min_size)
    elif cmd == 'primary_data':
        find_primary_data(args[0])
    elif cmd == 'report_solid':
        SolidDataDir(args[0]).report()
    elif cmd == 'match_solid':
        if len(args) < 2:
            sys.stderr.write("Need to supply a SOLiD data dir and at "
                             "least one analysis directory\n")
            sys.exit(1)
        SolidDataDir(args[0]).match_primary_data(*args[1:])
    elif cmd == 'symlinks':
        find_symlinks(args[0])
    elif cmd == 'md5sums':
        find_md5sums(args[0], options.outfile)
    elif cmd == 'duplicates':
        find_duplicates(*args)
    elif cmd == 'temp_files':
        find_tmp_files(args[0])
    elif cmd == 'set_permissions':
        DataDir(args[0]).set_permissions(mode=options.mode,
                                         group=options.group)
    elif cmd == 'compress':
        if len(args) < 2:
            sys.stderr.write("Need to supply a data dir and at least "
                             "one extension\n")
            sys.exit(1)
        compress_files(args[0], args[1:], dry_run=options.dry_run)
    elif cmd == 'related':
        find_related(args[0])
    elif cmd == 'shell':
        Shell(args[0]).cmdloop()