Пример #1
0
 def startJobTree(self, options):
     """Runs jobtree using the given options (see Stack.getDefaultOptions
     and Stack.addJobTreeOptions).
     """
     self.verifyJobTreeOptions(options)
     setLoggingFromOptions(options)
     options.jobTree = os.path.abspath(options.jobTree)
     if os.path.isdir(options.jobTree):
         config, batchSystem = reloadJobTree(options.jobTree)
     else:
         config, batchSystem = createJobTree(options)
         #Setup first job.
         command = self.makeRunnable(options.jobTree)
         memory = self.getMemory()
         cpu = self.getCpu()
         time = self.getRunTime()
         if memory != sys.maxint:
             if cpu != sys.maxint:
                 createFirstJob(command, config, memory=memory, cpu=cpu, time=time)
             else:
                 createFirstJob(command, config, memory=memory, time=time)
         else:
             if cpu != sys.maxint:
                 createFirstJob(command, config, cpu=cpu, time=time)
             else:
                 createFirstJob(command, config, time=time)
     loadEnvironment(config)
     return mainLoop(config, batchSystem)
Пример #2
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################

    parser = OptionParser()

    parser.add_option("--host", dest="host")
    parser.add_option("--port", dest="port")
    parser.add_option("--databaseDir", dest="databaseDir")
    parser.add_option("--databaseOptions", dest="databaseOptions")
    parser.add_option("--keysPerJob", dest="keysPerJob")
    parser.add_option("--totalJobs", dest="totalJobs")
    parser.add_option("--minRecordSize", dest="minRecordSize")
    parser.add_option("--maxRecordSize", dest="maxRecordSize")
    parser.add_option("--test",
                      dest="test",
                      action="store_true",
                      help="Run doctest unit tests")

    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    if options.test:
        _test()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    Stack(AddKeysPhase(options)).startJobTree(options)
Пример #3
0
def main():
    #Parse the inputs args/options
    parser = OptionParser(usage="usage: workingDir [options]",
                          version="%prog 0.1")
    options = Options()
    parser.add_option("--sequences",
                      dest="sequences",
                      help="Quoted list of fasta files containing sequences")
    parser.add_option("--alignments", dest="alignments", help="Cigar file ")
    addExpectationMaximisationOptions(parser, options)

    Stack.addJobTreeOptions(parser)
    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Expected no arguments, got %s arguments: %s" %
                           (len(args), " ".join(args)))

    #Log the inputs
    logger.info(
        "Got '%s' sequences, '%s' alignments file, '%s' output model and '%s' iterations of training"
        % (options.sequences, options.alignments, options.outputModel,
           options.iterations))

    #This line invokes jobTree
    i = Stack(
        Target.makeTargetFn(expectationMaximisationTrials,
                            args=(options.sequences, options.alignments,
                                  options.outputModel,
                                  options))).startJobTree(options)

    if i != 0:
        raise RuntimeError("Got failed jobs")
Пример #4
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("alignment", help="HAL alignment file")
    parser.add_argument("srcGenome", help="Reference genome")
    parser.add_argument("bedFile", help="Bed file (in ref coordinates)")
    parser.add_argument("destGenome", help="Genome to check contiguity in")
    parser.add_argument("outFile", help="Output BED file")
    parser.add_argument("--maxGap", help="maximum gap size to accept", 
                        default=100, type=int)
    parser.add_argument("--deletionGaps", help="care about deletion gaps",
                        default=False, action='store_true')
    parser.add_argument("--sliceNum", help="number of slices to create",
                        type=int, default=1)
    parser.add_argument("--maxIntronDiff", help="Maximum number of bases "
                        "that intron gaps are allowed to change by", type=int,
                        default=10000)
    parser.add_argument("--requiredMapFraction", help="Fraction of bases in "
                        "the query that need to map to the target to be "
                        "accepted", type=float, default=0.0)
    parser.add_argument("--printStats", help="instead of printing the "
                        "passing BED lines, print statistics",
                        action='store_true', default=False)
    Stack.addJobTreeOptions(parser)
    args = parser.parse_args()
    setLoggingFromOptions(args)
    result = Stack(Setup(args)).startJobTree(args)
    if result:
        raise RuntimeError("Jobtree has failed jobs.")

    return 0
Пример #5
0
def main():
    usage = "usage: %prog outputSequenceDir configXMLFile inputSequenceFastaFilesxN [options]"
    parser = OptionParser(usage=usage)
    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    if len(args) < 3:
        raise RuntimeError("Too few input arguments: %s" % " ".join(args))

    outputSequenceDir = args[0]
    configFile = args[1]
    inputSequences = args[2:]

    # Replace any constants
    configNode = ET.parse(configFile).getroot()
    if configNode.find("constants") != None:
        ConfigWrapper(configNode).substituteAllPredefinedConstantsWithLiterals()

    Stack(
        CactusPreprocessor(
            inputSequences, CactusPreprocessor.getOutputSequenceFiles(inputSequences, outputSequenceDir), configNode
        )
    ).startJobTree(options)
Пример #6
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("alignment", help="HAL alignment file")
    parser.add_argument("srcGenome", help="Reference genome")
    parser.add_argument("bedFile", help="Bed file (in ref coordinates)")
    parser.add_argument("destGenome", help="Genome to check contiguity in")
    parser.add_argument("outFile", help="Output BED file")
    parser.add_argument("--maxGap", help="maximum gap size to accept", 
                        default=100, type=int)
    parser.add_argument("--deletionGaps", help="care about deletion gaps",
                        default=False, action='store_true')
    parser.add_argument("--sliceNum", help="number of slices to create",
                        type=int, default=1)
    parser.add_argument("--maxIntronDiff", help="Maximum number of bases "
                        "that intron gaps are allowed to change by", type=int,
                        default=10000)
    parser.add_argument("--requiredMapFraction", help="Fraction of bases in "
                        "the query that need to map to the target to be "
                        "accepted", type=float, default=0.0)
    parser.add_argument("--printStats", help="instead of printing the "
                        "passing BED lines, print statistics",
                        action='store_true', default=False)
    Stack.addJobTreeOptions(parser)
    args = parser.parse_args()
    setLoggingFromOptions(args)
    result = Stack(Setup(args)).startJobTree(args)
    if result:
        raise RuntimeError("Jobtree has failed jobs.")

    return 0
Пример #7
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################

    parser = OptionParser()
 
    parser.add_option("--haplotypeSequences", dest="haplotypeSequences")
    parser.add_option("--newickTree", dest="newickTree")
    parser.add_option("--assembliesDir", dest="assembliesDir")
    parser.add_option("--outputDir", dest="outputDir")
    parser.add_option("--configFile", dest="configFile")
    parser.add_option("--minimumNsForScaffoldGap", dest="minimumNsForScaffoldGap")
    parser.add_option("--assemblyEventString", dest="assemblyEventString")
    parser.add_option("--haplotype1EventString", dest="haplotype1EventString")
    parser.add_option("--haplotype2EventString", dest="haplotype2EventString")
    parser.add_option("--contaminationEventString", dest="contaminationEventString")
    parser.add_option("--featureBedFiles", dest="featureBedFiles")
    parser.add_option("--geneBedFiles", dest="geneBedFiles")
    
    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    Stack(MakeAlignments(newickTree=options.newickTree, 
                         haplotypeSequences=options.haplotypeSequences.split(), 
                         assembliesDir=options.assembliesDir, 
                         outputDir=options.outputDir, 
                         configFile=options.configFile, 
                         options=options)).startJobTree(options)
    logger.info("Done with job tree")
Пример #8
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################

    parser = OptionParser()
    
    parser.add_option("--host", dest="host")
    parser.add_option("--port", dest="port")
    parser.add_option("--databaseDir", dest="databaseDir")
    parser.add_option("--databaseOptions", dest="databaseOptions")
    parser.add_option("--keysPerJob", dest="keysPerJob")
    parser.add_option("--totalJobs", dest="totalJobs")
    parser.add_option("--minRecordSize", dest="minRecordSize")
    parser.add_option("--maxRecordSize", dest="maxRecordSize")
    parser.add_option("--test", dest="test", action="store_true",
                      help="Run doctest unit tests")
    
    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    if options.test:
        _test()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))
    
    Stack(AddKeysPhase(options)).startJobTree(options)
def main():
    # init as dummy function
    cleanKtFn = lambda x,y:x
    stage = -1
    workDir = None
    try:
        parser = initParser()
        options, args = parser.parse_args()
        if len(args) == 0:
            parser.print_help()
            return 1
        if len(args) != 3:
            raise RuntimeError("Error parsing command line. Exactly 3 arguments are required but %d arguments were detected: %s" % (len(args), str(args)))
        
        if options.optionsFile != None:
            fileArgs = parseOptionsFile(options.optionsFile)
            options, args = parser.parse_args(fileArgs + sys.argv[1:])
            if len(args) != 3:
                raise RuntimeError("Error parsing options file.  Make sure all "
                                   "options have -- prefix")
        stage = 0
        setLoggingFromOptions(options)
        seqFile = SeqFile(args[0])
        workDir = args[1]
        outputHalFile = args[2]
        validateInput(workDir, outputHalFile, options)

        jtPath = os.path.join(workDir, "jobTree")
        stage = 1
        print "\nBeginning Alignment"
        system("rm -rf %s" % jtPath) 
        projWrapper = ProjectWrapper(options, seqFile, workDir)
        projWrapper.writeXml()
        jtCommands = getJobTreeCommands(jtPath, parser, options)
        runCactus(workDir, jtCommands, jtPath, options)
        cmd = 'jobTreeStatus --failIfNotComplete --jobTree %s &> /dev/null' %\
              jtPath
        system(cmd)

        stage = 2
        print "Beginning HAL Export"
        extractOutput(workDir, outputHalFile, options)
        print "Success.\n" "Temporary data was left in: %s\n" \
              % workDir
        
        return 0
    
    except RuntimeError, e:
        sys.stderr.write("Error: %s\n\n" % str(e))
        if stage >= 0 and os.path.isdir(workDir):
            sys.stderr.write("Temporary data was left in: %s\n" % workDir)
        if stage == 1:
            sys.stderr.write("More information can be found in %s\n" %
                             os.path.join(workDir, "cactus.log"))
        elif stage == 2:
            sys.stderr.write("More information can be found in %s\n" %
                             os.path.join(workDir, "cactus.log"))
        return -1
Пример #10
0
def runJobTreeScript(options):
    """Builds the basic job tree, or takes an existing one
    and runs the job tree master script.
    """
    setLoggingFromOptions(options)
    assert options.jobTree != None  #We need a job tree, or a place to create one
    if os.path.isdir(options.jobTree):
        config, batchSystem = reloadJobTree(options.jobTree)
    else:
        assert options.command != None
        config, batchSystem = createJobTree(options)
        #Setup first job.
        createFirstJob(options.command, config)
    loadEnvironment(config)
    return mainLoop(config, batchSystem)
Пример #11
0
def runJobTreeScript(options):
    """Builds the basic job tree, or takes an existing one
    and runs the job tree master script.
    """
    setLoggingFromOptions(options)
    assert options.jobTree != None #We need a job tree, or a place to create one
    if os.path.isdir(options.jobTree):
        config, batchSystem = reloadJobTree(options.jobTree)
    else:
        assert options.command != None
        config, batchSystem = createJobTree(options)
        #Setup first job.
        createFirstJob(options.command, config)
    loadEnvironment(config)
    return mainLoop(config, batchSystem)
Пример #12
0
 def startJobTree(self, options):
     """Runs jobtree using the given options (see Stack.getDefaultOptions
     and Stack.addJobTreeOptions).
     """
     self.verifyJobTreeOptions(options)
     setLoggingFromOptions(options)
     options.jobTree = absSymPath(options.jobTree)
     if os.path.isdir(options.jobTree):
         config, batchSystem = reloadJobTree(options.jobTree)
     else:
         config, batchSystem = createJobTree(options)
         #Setup first job.
         command = self.makeRunnable(options.jobTree)
         memory = self.getMemory()
         cpu = self.getCpu()
         createFirstJob(command, config, memory=memory, cpu=cpu)
     loadEnvironment(config)
     return mainLoop(config, batchSystem)
def main():
    ##########################################
    #Construct the arguments.
    ##########################################
    
    parser = OptionParser()
    parser.add_option("--outputDir", dest="outputDir")
    
    Stack.addJobTreeOptions(parser)
    
    options, args = parser.parse_args()
    setLoggingFromOptions(options)
    
    options.blanchetteRepeats = 1
    
    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))
    
    Stack(MakeAllAlignments(options)).startJobTree(options)
    logger.info("Done with job tree")
Пример #14
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################

    parser = OptionParser()

    parser.add_option("--haplotypeSequences", dest="haplotypeSequences")
    parser.add_option("--newickTree", dest="newickTree")
    parser.add_option("--outputDir", dest="outputDir")
    parser.add_option("--configFile", dest="configFile")
    parser.add_option("--referenceAlgorithms", dest="referenceAlgorithms")
    parser.add_option("--requiredSpecies", dest="requiredSpecies")
    parser.add_option("--singleCopySpecies", dest="singleCopySpecies")
    parser.add_option("--rangeOfMinimumBlockDegrees",
                      dest="rangeOfMinimumBlockDegrees")
    parser.add_option("--referenceSpecies", dest="referenceSpecies")
    parser.add_option("--minimumNsForScaffoldGap",
                      dest="minimumNsForScaffoldGap")
    parser.add_option("--blastAlignmentStrings", dest="blastAlignmentStrings")
    parser.add_option("--baseLevel", dest="baseLevel")
    parser.add_option("--maxNumberOfChains", dest="maxNumberOfChains")
    parser.add_option("--permutations", dest="permutations")
    parser.add_option("--theta", dest="theta")
    parser.add_option("--useSimulatedAnnealing", dest="useSimulatedAnnealing")
    parser.add_option("--sampleNumber", dest="sampleNumber")
    parser.add_option("--heldOutSequences", dest="heldOutSequences")
    parser.add_option("--outgroupEvent", dest="outgroupEvent")
    parser.add_option("--gapGamma", dest="gapGamma")

    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    Stack(MakeAlignments(options)).startJobTree(options)
    logger.info("Done with job tree")
Пример #15
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################
    
    parser = OptionParser()
 
    parser.add_option("--haplotypeSequences", dest="haplotypeSequences")
    parser.add_option("--newickTree", dest="newickTree")
    parser.add_option("--outputDir", dest="outputDir")
    parser.add_option("--configFile", dest="configFile")
    parser.add_option("--referenceAlgorithms", dest="referenceAlgorithms")
    parser.add_option("--requiredSpecies", dest="requiredSpecies")
    parser.add_option("--singleCopySpecies", dest="singleCopySpecies")
    parser.add_option("--rangeOfMinimumBlockDegrees", dest="rangeOfMinimumBlockDegrees")
    parser.add_option("--referenceSpecies", dest="referenceSpecies")
    parser.add_option("--minimumNsForScaffoldGap", dest="minimumNsForScaffoldGap")
    parser.add_option("--blastAlignmentStrings", dest="blastAlignmentStrings")
    parser.add_option("--baseLevel", dest="baseLevel")
    parser.add_option("--maxNumberOfChains", dest="maxNumberOfChains")
    parser.add_option("--permutations", dest="permutations")
    parser.add_option("--theta", dest="theta")
    parser.add_option("--useSimulatedAnnealing", dest="useSimulatedAnnealing")
    parser.add_option("--sampleNumber", dest="sampleNumber")
    parser.add_option("--heldOutSequences", dest="heldOutSequences")
    parser.add_option("--outgroupEvent", dest="outgroupEvent")
    parser.add_option("--gapGamma", dest="gapGamma")
    
    Stack.addJobTreeOptions(parser)
    
    options, args = parser.parse_args()
    setLoggingFromOptions(options)
    
    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))
    
    Stack(MakeAlignments(options)).startJobTree(options)
    logger.info("Done with job tree")
Пример #16
0
def main():
    ##########################################
    #Construct the arguments.
    ##########################################

    parser = OptionParser()

    parser.add_option("--haplotypeSequences", dest="haplotypeSequences")
    parser.add_option("--newickTree", dest="newickTree")
    parser.add_option("--assembliesDir", dest="assembliesDir")
    parser.add_option("--outputDir", dest="outputDir")
    parser.add_option("--configFile", dest="configFile")
    parser.add_option("--minimumNsForScaffoldGap",
                      dest="minimumNsForScaffoldGap")
    parser.add_option("--assemblyEventString", dest="assemblyEventString")
    parser.add_option("--haplotype1EventString", dest="haplotype1EventString")
    parser.add_option("--haplotype2EventString", dest="haplotype2EventString")
    parser.add_option("--contaminationEventString",
                      dest="contaminationEventString")
    parser.add_option("--featureBedFiles", dest="featureBedFiles")
    parser.add_option("--geneBedFiles", dest="geneBedFiles")

    Stack.addJobTreeOptions(parser)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    if len(args) != 0:
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    Stack(
        MakeAlignments(newickTree=options.newickTree,
                       haplotypeSequences=options.haplotypeSequences.split(),
                       assembliesDir=options.assembliesDir,
                       outputDir=options.outputDir,
                       configFile=options.configFile,
                       options=options)).startJobTree(options)
    logger.info("Done with job tree")
Пример #17
0
def main():
    #Parse the inputs args/options
    parser = OptionParser(usage="usage: workingDir [options]", version="%prog 0.1")
    options = Options()
    parser.add_option("--sequences", dest="sequences", help="Quoted list of fasta files containing sequences")
    parser.add_option("--alignments", dest="alignments", help="Cigar file ")
    addExpectationMaximisationOptions(parser, options)
    
    Stack.addJobTreeOptions(parser)
    options, args = parser.parse_args()
    setLoggingFromOptions(options)
    
    if len(args) != 0:
        raise RuntimeError("Expected no arguments, got %s arguments: %s" % (len(args), " ".join(args)))
    
    #Log the inputs
    logger.info("Got '%s' sequences, '%s' alignments file, '%s' output model and '%s' iterations of training" % (options.sequences, options.alignments, options.outputModel, options.iterations))

    #This line invokes jobTree  
    i = Stack(Target.makeTargetFn(expectationMaximisationTrials, args=(options.sequences, options.alignments, options.outputModel, options))).startJobTree(options) 
    
    if i != 0:
        raise RuntimeError("Got failed jobs")
Пример #18
0
def main():
    # init as dummy function
    cleanKtFn = lambda x,y:x
    stage = -1
    workDir = None
    try:
        parser = initParser()
        options, args = parser.parse_args()
        if (options.rootOutgroupDists is not None) \
        ^ (options.rootOutgroupPaths is not None):
            parser.error("--rootOutgroupDists and --rootOutgroupPaths must be " +
                         "provided together")
        if len(args) == 0:
            parser.print_help()
            return 1
        if len(args) != 3:
            raise RuntimeError("Error parsing command line. Exactly 3 arguments are required but %d arguments were detected: %s" % (len(args), str(args)))
        
        if options.optionsFile != None:
            fileArgs = parseOptionsFile(options.optionsFile)
            options, args = parser.parse_args(fileArgs + sys.argv[1:])
            if len(args) != 3:
                raise RuntimeError("Error parsing options file.  Make sure all "
                                   "options have -- prefix")
        stage = 0
        setLoggingFromOptions(options)
        seqFile = SeqFile(args[0])
        workDir = args[1]
        outputHalFile = args[2]
        validateInput(workDir, outputHalFile, options)

        jtPath = os.path.join(workDir, "jobTree")
        stage = 1
        print "\nBeginning Alignment"
        system("rm -rf %s" % jtPath) 
        projWrapper = ProjectWrapper(options, seqFile, workDir)
        projWrapper.writeXml()
        jtCommands = getJobTreeCommands(jtPath, parser, options)
        runCactus(workDir, jtCommands, jtPath, options)
        cmd = 'jobTreeStatus --failIfNotComplete --jobTree %s > /dev/null 2>&1 ' %\
              jtPath
        system(cmd)

        stage = 2
        print "Beginning HAL Export"
        extractOutput(workDir, outputHalFile, options)
        print "Success.\n" "Temporary data was left in: %s\n" \
              % workDir
        
        return 0
    
    except RuntimeError, e:
        sys.stderr.write("Error: %s\n\n" % str(e))
        if stage >= 0 and workDir is not None and os.path.isdir(workDir):
            sys.stderr.write("Temporary data was left in: %s\n" % workDir)
        if stage == 1:
            sys.stderr.write("More information can be found in %s\n" %
                             os.path.join(workDir, "cactus.log"))
        elif stage == 2:
            sys.stderr.write("More information can be found in %s\n" %
                             os.path.join(workDir, "cactus.log"))
        return -1
                        chrom, start, stop))
                continue
            test_ancestral_nodes(target, region_specific_conserved,
                                 accelerated_genomes, maf_path, region_bed,
                                 outf_handle)


def cat_results(target, args, paths):
    """
    Concatenates final phastcons output into one gff file.
    """
    fofn = os.path.join(target.getGlobalTempDir(), 'fofn')
    with open(fofn, 'w') as outf:
        for p in paths:
            outf.write(p + '\n')
    tmp_p = os.path.join(target.getGlobalTempDir(),
                         os.path.basename(args.out_bed) + '.tmp')
    cat_cmd = 'cat {} | xargs -n 50 cat > {}'.format(fofn, tmp_p)
    system(cat_cmd)
    os.rename(tmp_p, args.out_bed)


if __name__ == '__main__':
    from phast.run_acceleration_tests import *
    args = parse_args()
    setLoggingFromOptions(args)
    s = Stack(Target.makeTargetFn(extract_maf_wrapper, args=(args, )))
    i = s.startJobTree(args)
    if i != 0:
        raise RuntimeError("Got failed jobs")