def startJobTree(self, options): """Runs jobtree using the given options (see Stack.getDefaultOptions and Stack.addJobTreeOptions). """ self.verifyJobTreeOptions(options) setLoggingFromOptions(options) options.jobTree = os.path.abspath(options.jobTree) if os.path.isdir(options.jobTree): config, batchSystem = reloadJobTree(options.jobTree) else: config, batchSystem = createJobTree(options) #Setup first job. command = self.makeRunnable(options.jobTree) memory = self.getMemory() cpu = self.getCpu() time = self.getRunTime() if memory != sys.maxint: if cpu != sys.maxint: createFirstJob(command, config, memory=memory, cpu=cpu, time=time) else: createFirstJob(command, config, memory=memory, time=time) else: if cpu != sys.maxint: createFirstJob(command, config, cpu=cpu, time=time) else: createFirstJob(command, config, time=time) loadEnvironment(config) return mainLoop(config, batchSystem)
def main(): ########################################## #Construct the arguments. ########################################## parser = OptionParser() parser.add_option("--host", dest="host") parser.add_option("--port", dest="port") parser.add_option("--databaseDir", dest="databaseDir") parser.add_option("--databaseOptions", dest="databaseOptions") parser.add_option("--keysPerJob", dest="keysPerJob") parser.add_option("--totalJobs", dest="totalJobs") parser.add_option("--minRecordSize", dest="minRecordSize") parser.add_option("--maxRecordSize", dest="maxRecordSize") parser.add_option("--test", dest="test", action="store_true", help="Run doctest unit tests") Stack.addJobTreeOptions(parser) options, args = parser.parse_args() if options.test: _test() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args)) Stack(AddKeysPhase(options)).startJobTree(options)
def main(): #Parse the inputs args/options parser = OptionParser(usage="usage: workingDir [options]", version="%prog 0.1") options = Options() parser.add_option("--sequences", dest="sequences", help="Quoted list of fasta files containing sequences") parser.add_option("--alignments", dest="alignments", help="Cigar file ") addExpectationMaximisationOptions(parser, options) Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Expected no arguments, got %s arguments: %s" % (len(args), " ".join(args))) #Log the inputs logger.info( "Got '%s' sequences, '%s' alignments file, '%s' output model and '%s' iterations of training" % (options.sequences, options.alignments, options.outputModel, options.iterations)) #This line invokes jobTree i = Stack( Target.makeTargetFn(expectationMaximisationTrials, args=(options.sequences, options.alignments, options.outputModel, options))).startJobTree(options) if i != 0: raise RuntimeError("Got failed jobs")
def main(): parser = argparse.ArgumentParser() parser.add_argument("alignment", help="HAL alignment file") parser.add_argument("srcGenome", help="Reference genome") parser.add_argument("bedFile", help="Bed file (in ref coordinates)") parser.add_argument("destGenome", help="Genome to check contiguity in") parser.add_argument("outFile", help="Output BED file") parser.add_argument("--maxGap", help="maximum gap size to accept", default=100, type=int) parser.add_argument("--deletionGaps", help="care about deletion gaps", default=False, action='store_true') parser.add_argument("--sliceNum", help="number of slices to create", type=int, default=1) parser.add_argument("--maxIntronDiff", help="Maximum number of bases " "that intron gaps are allowed to change by", type=int, default=10000) parser.add_argument("--requiredMapFraction", help="Fraction of bases in " "the query that need to map to the target to be " "accepted", type=float, default=0.0) parser.add_argument("--printStats", help="instead of printing the " "passing BED lines, print statistics", action='store_true', default=False) Stack.addJobTreeOptions(parser) args = parser.parse_args() setLoggingFromOptions(args) result = Stack(Setup(args)).startJobTree(args) if result: raise RuntimeError("Jobtree has failed jobs.") return 0
def main(): usage = "usage: %prog outputSequenceDir configXMLFile inputSequenceFastaFilesxN [options]" parser = OptionParser(usage=usage) Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) < 3: raise RuntimeError("Too few input arguments: %s" % " ".join(args)) outputSequenceDir = args[0] configFile = args[1] inputSequences = args[2:] # Replace any constants configNode = ET.parse(configFile).getroot() if configNode.find("constants") != None: ConfigWrapper(configNode).substituteAllPredefinedConstantsWithLiterals() Stack( CactusPreprocessor( inputSequences, CactusPreprocessor.getOutputSequenceFiles(inputSequences, outputSequenceDir), configNode ) ).startJobTree(options)
def main(): ########################################## #Construct the arguments. ########################################## parser = OptionParser() parser.add_option("--haplotypeSequences", dest="haplotypeSequences") parser.add_option("--newickTree", dest="newickTree") parser.add_option("--assembliesDir", dest="assembliesDir") parser.add_option("--outputDir", dest="outputDir") parser.add_option("--configFile", dest="configFile") parser.add_option("--minimumNsForScaffoldGap", dest="minimumNsForScaffoldGap") parser.add_option("--assemblyEventString", dest="assemblyEventString") parser.add_option("--haplotype1EventString", dest="haplotype1EventString") parser.add_option("--haplotype2EventString", dest="haplotype2EventString") parser.add_option("--contaminationEventString", dest="contaminationEventString") parser.add_option("--featureBedFiles", dest="featureBedFiles") parser.add_option("--geneBedFiles", dest="geneBedFiles") Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args)) Stack(MakeAlignments(newickTree=options.newickTree, haplotypeSequences=options.haplotypeSequences.split(), assembliesDir=options.assembliesDir, outputDir=options.outputDir, configFile=options.configFile, options=options)).startJobTree(options) logger.info("Done with job tree")
def main(): # init as dummy function cleanKtFn = lambda x,y:x stage = -1 workDir = None try: parser = initParser() options, args = parser.parse_args() if len(args) == 0: parser.print_help() return 1 if len(args) != 3: raise RuntimeError("Error parsing command line. Exactly 3 arguments are required but %d arguments were detected: %s" % (len(args), str(args))) if options.optionsFile != None: fileArgs = parseOptionsFile(options.optionsFile) options, args = parser.parse_args(fileArgs + sys.argv[1:]) if len(args) != 3: raise RuntimeError("Error parsing options file. Make sure all " "options have -- prefix") stage = 0 setLoggingFromOptions(options) seqFile = SeqFile(args[0]) workDir = args[1] outputHalFile = args[2] validateInput(workDir, outputHalFile, options) jtPath = os.path.join(workDir, "jobTree") stage = 1 print "\nBeginning Alignment" system("rm -rf %s" % jtPath) projWrapper = ProjectWrapper(options, seqFile, workDir) projWrapper.writeXml() jtCommands = getJobTreeCommands(jtPath, parser, options) runCactus(workDir, jtCommands, jtPath, options) cmd = 'jobTreeStatus --failIfNotComplete --jobTree %s &> /dev/null' %\ jtPath system(cmd) stage = 2 print "Beginning HAL Export" extractOutput(workDir, outputHalFile, options) print "Success.\n" "Temporary data was left in: %s\n" \ % workDir return 0 except RuntimeError, e: sys.stderr.write("Error: %s\n\n" % str(e)) if stage >= 0 and os.path.isdir(workDir): sys.stderr.write("Temporary data was left in: %s\n" % workDir) if stage == 1: sys.stderr.write("More information can be found in %s\n" % os.path.join(workDir, "cactus.log")) elif stage == 2: sys.stderr.write("More information can be found in %s\n" % os.path.join(workDir, "cactus.log")) return -1
def runJobTreeScript(options): """Builds the basic job tree, or takes an existing one and runs the job tree master script. """ setLoggingFromOptions(options) assert options.jobTree != None #We need a job tree, or a place to create one if os.path.isdir(options.jobTree): config, batchSystem = reloadJobTree(options.jobTree) else: assert options.command != None config, batchSystem = createJobTree(options) #Setup first job. createFirstJob(options.command, config) loadEnvironment(config) return mainLoop(config, batchSystem)
def startJobTree(self, options): """Runs jobtree using the given options (see Stack.getDefaultOptions and Stack.addJobTreeOptions). """ self.verifyJobTreeOptions(options) setLoggingFromOptions(options) options.jobTree = absSymPath(options.jobTree) if os.path.isdir(options.jobTree): config, batchSystem = reloadJobTree(options.jobTree) else: config, batchSystem = createJobTree(options) #Setup first job. command = self.makeRunnable(options.jobTree) memory = self.getMemory() cpu = self.getCpu() createFirstJob(command, config, memory=memory, cpu=cpu) loadEnvironment(config) return mainLoop(config, batchSystem)
def main(): ########################################## #Construct the arguments. ########################################## parser = OptionParser() parser.add_option("--outputDir", dest="outputDir") Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) options.blanchetteRepeats = 1 if len(args) != 0: raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args)) Stack(MakeAllAlignments(options)).startJobTree(options) logger.info("Done with job tree")
def main(): ########################################## #Construct the arguments. ########################################## parser = OptionParser() parser.add_option("--haplotypeSequences", dest="haplotypeSequences") parser.add_option("--newickTree", dest="newickTree") parser.add_option("--outputDir", dest="outputDir") parser.add_option("--configFile", dest="configFile") parser.add_option("--referenceAlgorithms", dest="referenceAlgorithms") parser.add_option("--requiredSpecies", dest="requiredSpecies") parser.add_option("--singleCopySpecies", dest="singleCopySpecies") parser.add_option("--rangeOfMinimumBlockDegrees", dest="rangeOfMinimumBlockDegrees") parser.add_option("--referenceSpecies", dest="referenceSpecies") parser.add_option("--minimumNsForScaffoldGap", dest="minimumNsForScaffoldGap") parser.add_option("--blastAlignmentStrings", dest="blastAlignmentStrings") parser.add_option("--baseLevel", dest="baseLevel") parser.add_option("--maxNumberOfChains", dest="maxNumberOfChains") parser.add_option("--permutations", dest="permutations") parser.add_option("--theta", dest="theta") parser.add_option("--useSimulatedAnnealing", dest="useSimulatedAnnealing") parser.add_option("--sampleNumber", dest="sampleNumber") parser.add_option("--heldOutSequences", dest="heldOutSequences") parser.add_option("--outgroupEvent", dest="outgroupEvent") parser.add_option("--gapGamma", dest="gapGamma") Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args)) Stack(MakeAlignments(options)).startJobTree(options) logger.info("Done with job tree")
def main(): ########################################## #Construct the arguments. ########################################## parser = OptionParser() parser.add_option("--haplotypeSequences", dest="haplotypeSequences") parser.add_option("--newickTree", dest="newickTree") parser.add_option("--assembliesDir", dest="assembliesDir") parser.add_option("--outputDir", dest="outputDir") parser.add_option("--configFile", dest="configFile") parser.add_option("--minimumNsForScaffoldGap", dest="minimumNsForScaffoldGap") parser.add_option("--assemblyEventString", dest="assemblyEventString") parser.add_option("--haplotype1EventString", dest="haplotype1EventString") parser.add_option("--haplotype2EventString", dest="haplotype2EventString") parser.add_option("--contaminationEventString", dest="contaminationEventString") parser.add_option("--featureBedFiles", dest="featureBedFiles") parser.add_option("--geneBedFiles", dest="geneBedFiles") Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args)) Stack( MakeAlignments(newickTree=options.newickTree, haplotypeSequences=options.haplotypeSequences.split(), assembliesDir=options.assembliesDir, outputDir=options.outputDir, configFile=options.configFile, options=options)).startJobTree(options) logger.info("Done with job tree")
def main(): #Parse the inputs args/options parser = OptionParser(usage="usage: workingDir [options]", version="%prog 0.1") options = Options() parser.add_option("--sequences", dest="sequences", help="Quoted list of fasta files containing sequences") parser.add_option("--alignments", dest="alignments", help="Cigar file ") addExpectationMaximisationOptions(parser, options) Stack.addJobTreeOptions(parser) options, args = parser.parse_args() setLoggingFromOptions(options) if len(args) != 0: raise RuntimeError("Expected no arguments, got %s arguments: %s" % (len(args), " ".join(args))) #Log the inputs logger.info("Got '%s' sequences, '%s' alignments file, '%s' output model and '%s' iterations of training" % (options.sequences, options.alignments, options.outputModel, options.iterations)) #This line invokes jobTree i = Stack(Target.makeTargetFn(expectationMaximisationTrials, args=(options.sequences, options.alignments, options.outputModel, options))).startJobTree(options) if i != 0: raise RuntimeError("Got failed jobs")
def main(): # init as dummy function cleanKtFn = lambda x,y:x stage = -1 workDir = None try: parser = initParser() options, args = parser.parse_args() if (options.rootOutgroupDists is not None) \ ^ (options.rootOutgroupPaths is not None): parser.error("--rootOutgroupDists and --rootOutgroupPaths must be " + "provided together") if len(args) == 0: parser.print_help() return 1 if len(args) != 3: raise RuntimeError("Error parsing command line. Exactly 3 arguments are required but %d arguments were detected: %s" % (len(args), str(args))) if options.optionsFile != None: fileArgs = parseOptionsFile(options.optionsFile) options, args = parser.parse_args(fileArgs + sys.argv[1:]) if len(args) != 3: raise RuntimeError("Error parsing options file. Make sure all " "options have -- prefix") stage = 0 setLoggingFromOptions(options) seqFile = SeqFile(args[0]) workDir = args[1] outputHalFile = args[2] validateInput(workDir, outputHalFile, options) jtPath = os.path.join(workDir, "jobTree") stage = 1 print "\nBeginning Alignment" system("rm -rf %s" % jtPath) projWrapper = ProjectWrapper(options, seqFile, workDir) projWrapper.writeXml() jtCommands = getJobTreeCommands(jtPath, parser, options) runCactus(workDir, jtCommands, jtPath, options) cmd = 'jobTreeStatus --failIfNotComplete --jobTree %s > /dev/null 2>&1 ' %\ jtPath system(cmd) stage = 2 print "Beginning HAL Export" extractOutput(workDir, outputHalFile, options) print "Success.\n" "Temporary data was left in: %s\n" \ % workDir return 0 except RuntimeError, e: sys.stderr.write("Error: %s\n\n" % str(e)) if stage >= 0 and workDir is not None and os.path.isdir(workDir): sys.stderr.write("Temporary data was left in: %s\n" % workDir) if stage == 1: sys.stderr.write("More information can be found in %s\n" % os.path.join(workDir, "cactus.log")) elif stage == 2: sys.stderr.write("More information can be found in %s\n" % os.path.join(workDir, "cactus.log")) return -1
chrom, start, stop)) continue test_ancestral_nodes(target, region_specific_conserved, accelerated_genomes, maf_path, region_bed, outf_handle) def cat_results(target, args, paths): """ Concatenates final phastcons output into one gff file. """ fofn = os.path.join(target.getGlobalTempDir(), 'fofn') with open(fofn, 'w') as outf: for p in paths: outf.write(p + '\n') tmp_p = os.path.join(target.getGlobalTempDir(), os.path.basename(args.out_bed) + '.tmp') cat_cmd = 'cat {} | xargs -n 50 cat > {}'.format(fofn, tmp_p) system(cat_cmd) os.rename(tmp_p, args.out_bed) if __name__ == '__main__': from phast.run_acceleration_tests import * args = parse_args() setLoggingFromOptions(args) s = Stack(Target.makeTargetFn(extract_maf_wrapper, args=(args, ))) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")