def main(): parser = argparse.ArgumentParser() parser.add_argument("--genome", required=True) parser.add_argument("--refGenome", required=True) parser.add_argument("--refTranscriptFasta", required=True) parser.add_argument("--targetGenomeFasta", required=True) parser.add_argument("--outDb", default="cgp_cds_metrics.db") parser.add_argument("--compAnnPath", required=True) gp_group = parser.add_mutually_exclusive_group(required=True) gp_group.add_argument("--cgpGp") gp_group.add_argument("--consensusGp") Stack.addJobTreeOptions(parser) args = parser.parse_args() out_db = os.path.join(args.compAnnPath, args.outDb) if args.cgpGp is not None: gp = args.cgpGp mode = "cgp" chunk_size = 15 # smaller chunk size because we will do more alignments per transcript else: gp = args.consensusGp mode = "consensus" chunk_size = 40 s = Stack( Target.makeTargetFn(align_gp, args=[ args.genome, args.refGenome, args.refTranscriptFasta, args.targetGenomeFasta, gp, mode, out_db, args.compAnnPath, chunk_size ])) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def main(): parser = argparse.ArgumentParser() parser.add_argument("--genome", required=True) parser.add_argument("--database", required=True) parser.add_argument("--hintsDir", required=True) parser.add_argument("--fasta", required=True) parser.add_argument("--filterTissues", nargs="+") parser.add_argument("--filterCenters", nargs="+") bamfiles = parser.add_mutually_exclusive_group(required=True) bamfiles.add_argument("--bamFiles", nargs="+", help="bamfiles being used", dest="bams") bamfiles.add_argument("--bamFofn", help="File containing list of bamfiles", dest="bams") Stack.addJobTreeOptions(parser) args = parser.parse_args() # below is an ugly hack to filter out tissues/centers by checking for the words in the file path if not isinstance(args.bams, list): if not os.path.exists(args.bams): raise RuntimeError("ERROR: bamFofn does not exist.") bams = {x.rstrip() for x in open(args.bams)} args.bams = bams else: args.bams = set(args.bams) for to_remove_list in [args.filterTissues, args.filterCenters]: if isinstance(to_remove_list, list): to_remove = set() for x in to_remove_list: for b in args.bams: if x in b: to_remove.add(b) args.bams -= to_remove s = Stack(Target.makeTargetFn(main_hints_fn, memory=8 * 1024 ** 3, args=[args.bams, args.database, args.genome, args.fasta, args.hintsDir])) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def main(): args = parse_args() args.target_genomes = extract_model_tree(args.model) - set( [args.ref_genome]) args.msa_split_options = " ".join( ['--windows', args.windows, '--between-blocks', args.between_blocks]) s = Stack(Target.makeTargetFn(dless_pipeline_wrapper, args=(args, ))) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def main(): args = parse_args() if args.target_genomes is None: args.target_genomes = extract_model_tree(args.model) - set( [args.ref_genome]) args.msa_split_options = ' '.join([ '--windows', args.windows, '--between-blocks', args.between_blocks, '--min-informative', args.min_informative ]) s = Stack(Target.makeTargetFn(subset_hal_pipeline, args=(args, ))) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def main(): parser = build_parser() Stack.addJobTreeOptions(parser) args = parser.parse_args() if args.mode == "reference": s = Stack(Target.makeTargetFn(main_ref_fn, args=[args.comparativeAnnotationDir, args.gencode, args.genome, args.outDir, args.filterChroms])) elif args.mode == "transMap": s = Stack(Target.makeTargetFn(main_fn, args=[args.comparativeAnnotationDir, args.gencode, args.genome, args.refGenome, args.outDir, args.filterChroms])) else: s = Stack(Target.makeTargetFn(main_augustus_fn, args=[args.comparativeAnnotationDir, args.gencode, args.genome, args.outDir, args.filterChroms])) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def main(): ## Make sure we're in the right type of directory assert os.path.exists("jobs.list") assert os.path.exists("jobsEM.list") assert os.path.exists("config.txt") assert os.path.exists("configEM.txt") assert os.path.exists("params0.txt") assert commandAvailable(collectParamsExec) assert commandAvailable(mergeSwarm) assert commandAvailable(mergeMerge) ## ## Parse options ## parser = OptionParser() Stack.addJobTreeOptions(parser) # so that the stack will work parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" assert len(args) == 0 or len(args) == 1 tolerance = 0.001 if len(args) == 1: tolerance = float(args[0]) logging.info("options: " + str(options)) ## ## Run ## logging.info("starting first EM iteration") s = Stack(ExpectationIteration(0, tolerance, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print("%d jobs failed" % failed) else: logging.info("Run complete!")
def main(): ## Make sure we're in the right type of directory assert os.path.exists("jobs.list") assert os.path.exists("jobsEM.list") assert os.path.exists("config.txt") assert os.path.exists("configEM.txt") assert os.path.exists("params0.txt") assert commandAvailable(collectParamsExec) assert commandAvailable(mergeSwarm) assert commandAvailable(mergeMerge) ## ## Parse options ## parser = OptionParser() Stack.addJobTreeOptions(parser) # so that the stack will work parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" assert len(args) == 0 or len(args) == 1 tolerance = 0.001 if len(args) == 1: tolerance = float(args[0]) logger.info("options: " + str(options)) ## ## Run ## logger.info("starting first EM iteration") s = Stack(ExpectationIteration(0, tolerance, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!")
def start(self): """ probably need one of these in experiment """ if self.resume == 0: self.createAnalysisDir() stack = Stack(self.pipeline) options = stack.getDefaultOptions() options.jobTree = self.dir + 'jobTreeRun' if self.resume != 0 and os.path.exists(self.dir + 'jobTreeRun'): shutil.rmtree(self.dir + 'jobTreeRun') options.logLevel = 'INFO' # need to set batch system, big mem/cpu batches print 'starting jobTree' i = stack.startJobTree(options) print "success!!!"
def main(): ## check for fresh run if os.path.exists('.jobTree'): logger('WARNING: .jobTree directory found, remove it first to start a fresh run\n') ## parse arguments parser = OptionParser(usage = '%prog [options]') Stack.addJobTreeOptions(parser) parser.add_option('--jobFile', help='Add as child of jobFile rather than new jobTree') parser.add_option('-d', '--data', dest='data_file', default=None) parser.add_option('-p', '--phenotype', dest='phenotype_file', default=None) options, args = parser.parse_args() logger('Using Batch System : %s\n' % (options.batchSystem)) if len(args) == 1: if args[0] == 'clean': command = 'rm -rf .jobTree analysis' logger(command) os.system(command) sys.exit(0) assert(len(args) == 0) ## set parameters parameters = Parameters() ## run s = Stack(branchSignatures(options.data_file, options.phenotype_file, parameters, os.getcwd().rstrip('/'))) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: logger('%d jobs failed\n' % failed) else: os.system('rm -rf .lastjobTree') os.system('mv .jobTree .lastjobTree')
def main(): args = parse_args() if args.target_genomes is None: args.target_genomes = extract_model_tree(args.model) - set( [args.ref_genome]) else: args.target_genomes = set(args.target_genomes) - set([args.ref_genome]) args.msa_split_options = " ".join([ '--windows', args.windows, '--between-blocks', args.between_blocks, '--min-informative', args.min_informative ]) args.phastcons_options = " ".join([ '--target-coverage', args.target_coverage, '--expected-length', args.expected_length ]) s = Stack(Target.makeTargetFn(phastcons_pipeline_wrapper, args=(args, ))) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
for l in r: l = l.split() tot += int(l[-1]) - int(l[-2]) outf.write('\t'.join( map(str, [chrom, start, stop, format_ratio(tot, length)])) + '\n') def cat_results(target, args, paths): """ Concatenates final scores output into one bed file. """ fofn = os.path.join(target.getGlobalTempDir(), 'fofn') with open(fofn, 'w') as outf: for p in paths: outf.write(p + '\n') tmp_p = os.path.join(target.getGlobalTempDir(), os.path.basename(args.out_bed) + '.tmp') cat_cmd = 'cat {} | xargs -n 50 cat > {}'.format(fofn, tmp_p) system(cat_cmd) os.rename(tmp_p, args.out_bed) if __name__ == '__main__': from phast.find_single_copy_regions import * args = parse_args() s = Stack(Target.makeTargetFn(single_copy_wrapper, args=(args, ))) i = s.startJobTree(args) if i != 0: raise RuntimeError("Got failed jobs")
def jtFitness(): ## parse arguments parser = OptionParser() Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-d", "--dogma", dest="dogmaPath", default="") parser.add_option("-p", "--pathway", dest="pathwayPath", default="") parser.add_option("-b", "--boundaries", dest="discBound", default="") parser.add_option("-s", "--shuffle", dest="shuffleNode", default="") parser.add_option("-n", "--nulls", dest="nNulls", default="10") parser.add_option("-t", "--storedparam", dest="paramFile", default="") options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" evidList = args if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) if len(options.discBound) == 0: disc = "0.3333;0.6667" else: disc = options.discBound if len(options.dogmaPath) == 0: dogma = "%s/%s" % (dogmaDir, dogmaDefault) else: dogma = options.dogmaPath if not dogma.startswith("/"): dogma = "%s/%s" % (os.getcwd(), dogma) if len(options.pathwayPath) == 0: pathway = "%s/%s" % (pathwayDir, pathwayDefault) else: pathway = options.pathwayPath if not pathway.startswith("/"): pathway = "%s/%s" % (os.getcwd(), pathway) if len(options.shuffleNode) == 0: shuffleNode = "NULL" else: shuffleNode = options.shuffleNode nShuffle = int(options.nNulls) if len(options.paramFile) == 0: paramFile = None else: paramFile = options.paramFile ## clean if len(args) == 1: if args[0] == "clean": print "rm -rf .jobTree fold*" os.system("rm -rf .jobTree fold*") sys.exit(0) ## run logger.info("options: " + str(options)) s = Stack(branchFolds(" ".join(evidList), disc, paramFile, paradigmExec, inferSpec, dogma, pathway, shuffleNode, nShuffle, mFolds, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def main(): ## parse arguments parser = OptionParser(usage="%prog [options] network IPL-matrix features") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-w", "--workdir", dest="workdir", help="Common Work directory", default="./") parser.add_option("-i", "--ipl", dest="iplFile", default=None) parser.add_option("-p", "--pathway", dest="pathwayZip", default=None) parser.add_option("-c", "--phenotype", dest="phenotypeFile", default=None) parser.add_option("-o", "--oz", dest="outputZip", default=None) parser.add_option("-s", "--score", dest="scoreFile", default=None) parser.add_option("-f", "--filter", dest="filterParams", default="0.0;0.0") parser.add_option("-b", "--background", dest="nBackground", default="0") options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" ## clean if len(args) == 1: if args[0] == "clean": print "rm -rf real* null* OCCAM__* LAYOUT background.R .jobTree" system("rm -rf real* null* OCCAM__* LAYOUT background.R .jobTree") sys.exit(0) ## parse arguments assert ((len(args) == 0) or (len(args) == 2) or (len(args) == 3)) if len(args) == 0: pathwayZip = options.pathwayZip if options.pathwayZip is not None else basepathway pathwayLib = os.path.join(options.workdir, "pathway") system("unzip %s -d %s" % (pathwayZip, pathwayLib)) paradigmPathway = None for file in os.listdir(pathwayLib): if file.endswith("_pathway.tab"): paradigmPathway = "%s/%s" % (pathwayLib, file) break scoreFile = None phenotypeFile = options.phenotypeFile dataFile = options.iplFile sampleList = [] for sample in retColumns(dataFile): if not sample.startswith("na_iter"): sampleList.append(sample) filterParams = options.filterParams nNulls = int(options.nBackground) outputZip = options.outputZip assert (os.path.exists(paradigmPathway)) assert (os.path.exists(phenotypeFile)) assert (os.path.exists(dataFile)) elif len(args) == 2: paradigmPathway = args[0] scoreFile = args[1] phenotypeFile = None dataFile = None sampleList = None filterParams = options.filterParams nNulls = 0 outputZip = options.outputZip assert (os.path.exists(paradigmPathway)) assert (os.path.exists(scoreFile)) elif len(args) == 3: paradigmPathway = args[0] scoreFile = None phenotypeFile = args[2] dataFile = args[1] sampleList = [] for sample in retColumns(dataFile): if not sample.startswith("na_iter"): sampleList.append(sample) filterParams = options.filterParams nNulls = int(options.nBackground) outputZip = options.outputZip assert (os.path.exists(paradigmPathway)) assert (os.path.exists(phenotypeFile)) assert (os.path.exists(dataFile)) ## run logger.info("options: " + str(options)) logger.info("starting make") writeScripts() s = Stack( prepareOCCAM(paradigmPathway, scoreFile, phenotypeFile, None, dataFile, sampleList, filterParams, nNulls, outputZip, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def as_main(): ## check for fresh run if os.path.exists('.jobTree'): logger('WARNING: .jobTree directory found, remove it first to start a fresh run\n') ## parse arguments parser = OptionParser(usage = '%prog [options] paradigm_directory model_directory') Stack.addJobTreeOptions(parser) parser.add_option('--jobFile', help='Add as child of jobFile rather than new jobTree') parser.add_option('-n', '--nulls', dest='nulls', default=0) parser.add_option('-b', '--batchsize', dest='batch_size', default=50) parser.add_option('-y', '--public', action='store_true', dest='paradigm_public', default=False) parser.add_option('-g', '--galaxy', dest='galaxy_run', action='store_true', default=False) options, args = parser.parse_args() logger('Using Batch System : %s\n' % (options.batchSystem)) if len(args) == 1: if args[0] == 'clean': command = 'rm -rf .jobTree analysis' logger(command) os.system(command) sys.exit(0) assert(len(args) == 2) paradigm_directory = os.path.abspath(args[0]) model_directory = os.path.abspath(args[1]) ## set Galaxy Executables global paradigm_executable, circleplot_executable if options.galaxy_run: paradigm_executable = os.path.join(base_directory, 'paradigm') ## set Paradigm files paradigm_setup = ParadigmSetup(paradigm_directory, nulls = int(options.nulls), batch_size = int(options.batch_size), public = options.paradigm_public) ## gather models model_list = [] for file in os.listdir(model_directory): if os.path.exists('%s/%s/upstream_pathway.tab' % (model_directory, file)) and os.path.exists('%s/%s/downstream_pathway.tab' % (model_directory, file)): model = Model('%s/%s' % (model_directory, file)) model.setSeed() model_list.append(model) ## run s = Stack(queueModels(model_list, paradigm_setup, os.getcwd().rstrip('/'))) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = './.jobTree' failed = s.startJobTree(options) #failed = '' #queueModels(model_list, paradigm_setup, os.getcwd().rstrip('/')).run() if failed: logger('%d jobs failed\n' % failed) else: os.system('rm -rf .lastjobTree') os.system('mv .jobTree .lastjobTree')
def wrapParadigm(): ## parse arguments parser = OptionParser( usage="%prog [options] attachment file:path [attachment file:path ...]" ) Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-w", "--workdir", dest="workdir", help="Common Work directory", default="./") parser.add_option("-n", "--nulls", dest="nulls", help="Number of Null Samples", default="5") parser.add_option("-d", "--dogma", dest="dogma", help="Path to PARADIGM Dogma Specification", default=basedogma) parser.add_option("-i", "--imap", dest="imap", help="Path to PARADIGM Interaction Map Specification", default=baseimap) parser.add_option("-t", "--storedparam", dest="param", help="Initial Parameter Starting Point", default=baseparams) parser.add_option("--xgmml", dest="xgmml", help="Path to PARADIGM Pathway XGMML file", default=None) parser.add_option("-p", "--pathway", dest="pathway", help="Path to PARADIGM Pathway Specification", default=basepathway) parser.add_option("-b", "--boundaries", dest="disc", help="Data Discretization Bounds", default="0.33;0.67") parser.add_option("-s", "--skipem", action="store_false", dest="em", help="Skip Running EM", default=True) parser.add_option("--lb-max", dest="lb_max", help="Loopy Belief Max iterations", default=10000) parser.add_option("-o", "--output", dest="output_paradigm", help="Unfiltered Output", default="paradigm.output") parser.add_option("--op", "--output-params", dest="output_params", help="Parameter Output", default=None) parser.add_option("--oc", "--output-config", dest="output_config", help="Config Output", default=None) parser.add_option("--of", "--output-files", dest="output_files", help="Output Files", default=None) parser.add_option("-z", dest="private_paradigm", help="This is such bullshit", action="store_true", default=False) options, args = parser.parse_args() logging.info("options: " + str(options)) print "Using Batch System '" + options.batchSystem + "'" evidList = [] for i, element in enumerate(args): if i % 2 == 1: (fileType, filePath) = args[i].split(":") evidList.append("%s:%s" % (fileType, os.path.abspath(filePath))) else: evidList.append(args[i]) if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) workdir = os.path.abspath(options.workdir) if not os.path.exists(workdir): os.makedirs(workdir) nullBatches = int(options.nulls) dogma = os.path.abspath(options.dogma) imap = os.path.abspath(options.imap) params = os.path.abspath(options.param) disc = options.disc paramFile = os.path.abspath(options.param) runEM = options.em if not os.path.exists(workdir): os.makedirs(workdir) dogmaLib = os.path.join(workdir, "dogma") pathwayLib = os.path.join(workdir, "pathway") os.makedirs(dogmaLib) os.makedirs(pathwayLib) shutil.copy(dogma, dogmaLib) shutil.copy(imap, dogmaLib) if options.xgmml: pathway = os.path.join(pathwayLib, "pid_tmp_pathway.tab") system("%s %s %s %s" % (sys.executable, xgmmlExec, options.xgmml, pathway)) else: pathway = os.path.abspath(options.pathway) shutil.copy(pathway, os.path.join(pathwayLib, "pid_tmp_pathway.tab")) ## run logging.info("starting prepare") argSpec = inferSpec % (options.lb_max) s = Stack( prepareParadigm(evidSpec=" ".join(evidList), disc=disc, paramFile=paramFile, nullBatches=nullBatches, paradigmExec=paradigmExec, inferSpec=argSpec, dogmaLib=dogmaLib, pathwayLib=pathwayLib, em=runEM, directory=workdir, private_paradigm=options.private_paradigm)) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print("%d jobs failed" % failed) else: shutil.copy( os.path.join(options.workdir, "merge_merged_unfiltered.all.tab"), options.output_paradigm) if options.output_params is not None: shutil.copy(os.path.join(options.workdir, "params.txt"), options.output_params) if options.output_config is not None: shutil.copy(os.path.join(options.workdir, "config.txt"), options.output_config) if options.output_files is not None: system("zip -r outputFiles.zip outputFiles") shutil.copy(os.path.join(options.workdir, "outputFiles.zip"), options.output_files) logging.info("Run complete!") if os.path.exists(".lastjobTree"): system("rm -rf .lastjobTree") if os.path.exists(".jobTree"): system("mv .jobTree .lastjobTree")
def wrapParadigm(): ## parse arguments parser = OptionParser(usage = "%prog [options] attachment file:path [attachment file:path ...]") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-w", "--workdir", dest="workdir", help="Common Work directory", default="./") parser.add_option("-n", "--nulls", dest="nulls", help="Number of Null Samples", default="5") parser.add_option("-d", "--dogma", dest="dogmazip", help="Path to PARADIGM Dogma Specification", default=basedogma) parser.add_option("-p", "--pathway", dest="pathwayzip", help="Path to PARADIGM Pathway Specification", default=basepathway) parser.add_option("-b", "--boundaries", dest="disc", help="Data Discretization Bounds", default="0.33;0.67") parser.add_option("-t", "--storedparam", dest="param", help="Initial Parameter Starting Point", default=None) parser.add_option("-s", "--skipem", action="store_false", dest="em", help="Skip Running EM", default=True) parser.add_option("--fr", "--filter-real", dest="filtered_real", help="Filtered Output", default=None) parser.add_option("--fa", "--filter-all", dest="filtered_all", help="Filtered Output", default=None) parser.add_option("--ur", "--unfilter-real", dest="unfiltered_real", help="Filtered Output", default=None) parser.add_option("--ua", "--unfilter-all", dest="unfiltered_all", help="Filtered Output", default=None) options, args = parser.parse_args() logger.info("options: " + str(options)) print "Using Batch System '" + options.batchSystem + "'" evidList = [] for i, element in enumerate(args): if i % 2 == 1: (fileType, filePath) = args[i].split(":") evidList.append("%s:%s" % (fileType, os.path.abspath(filePath))) else: evidList.append(args[i]) if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) workdir = os.path.abspath(options.workdir) nullBatches = int(options.nulls) dogmaZip=os.path.abspath(options.dogmazip) pathwayZip=os.path.abspath(options.pathwayzip) disc=options.disc paramFile=os.path.abspath(options.param) if options.param is not None else None runEM = options.em dogmaLib = os.path.join(workdir, "dogma") pathwayLib = os.path.join(workdir, "pathway") system("unzip %s -d %s" % (dogmaZip, dogmaLib)) system("unzip %s -d %s" % (pathwayZip, pathwayLib)) ## run logger.info("starting prepare") s = Stack(prepareParadigm(" ".join(evidList), disc, paramFile, nullBatches, paradigmExec, inferSpec, dogmaLib, pathwayLib, runEM, workdir)) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def main(): ## configure logger formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s') file_handler = logging.FileHandler('signature.log') file_handler.setFormatter(formatter) file_handler.setLevel(logging.INFO) stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) stream_handler.setLevel(logging.ERROR) logger.addHandler(file_handler) logger.addHandler(stream_handler) ## check for fresh run if os.path.exists(".jobTree"): logger.warning("WARNING: '.jobTree' directory found, remove it first to start a fresh run\n") ## parse arguments parser = OptionParser(usage = "%prog [options] data_matrix phenotype_matrix") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as child of jobFile rather than new jobTree") parser.add_option("-b", "--bootstrap", dest = "bootstrap_size", default = 0, help = "Number of bootstrap samples to estimate subnetwork robustness") parser.add_option("-n", "--null", dest = "null_size", default = 0, help = "Number of null samples to estimate subnetwork significance") parser.add_option("-p", "--permute", dest = "null_permute", default = "paradigm", help = "Permutation method for generation of null samples") parser.add_option("-m", "--method", dest = "signature_method", default = "sam", help = "Differential method for computing signatures") parser.add_option("-z", "--seed", dest = "seed", default = None, help = "Random seed used for bootstrap and null generation") options, args = parser.parse_args() logger.info("options: %s" % (str(options))) print "Using Batch System '%s'" % (options.batchSystem) if len(args) != 2: logger.error("ERROR: incorrect number of arguments\n") sys.exit(1) data_file = os.path.abspath(args[0]) phenotype_file = os.path.abspath(args[1]) ## set parameters parameters = Parameters(signature_method = options.signature_method, bootstrap_size = int(options.bootstrap_size), null_size = int(options.null_size), null_method = options.null_permute, random_seed = options.seed) ## run s = Stack(branchSignatures(data_file, phenotype_file, parameters, os.getcwd().rstrip("/"))) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" jobtree_dir = options.jobTree.rstrip("/") lasttree_dir = jobtree_dir + "_previous" failed = s.startJobTree(options) if failed: logger.warning("WARNING: %d jobs failed" % (failed)) else: logger.info("Run complete!") if os.path.exists(lasttree_dir): shutil.rmtree(lasttree_dir) if os.path.exists(jobtree_dir): shutil.move(jobtree_dir, lasttree_dir)
def main(): ## check for fresh run if os.path.exists(".jobTree"): print "WARNING: .jobTree directory detected, remove it first to start a fresh run" ## parse arguments parser = OptionParser(usage = "%prog [options] paradigmDir mutFile ") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as child of jobFile rather than a new jobTree") parser.add_option("-s", "--samples", dest="includeSamples", default="") parser.add_option("-f", "--features", dest="includeFeatures", default="") options, args = parser.parse_args() print "Using Batch System '%s'" % (options.batchSystem) if len(args) == 1: if args[0] == "clean": cmd = "rm -rf .jobTree analysis *.R" print cmd os.system(cmd) sys.exit(0) assert len(args) == 2 paradigmDir = os.path.abspath(args[0]) mutFile = args[1] sampleFile = options.includeSamples featureFile = options.includeFeatures ## paramMap paramMap = {} paramMap["dist"] = [2] paramMap["thresh"] = [0.68] paramMap["inc"] = [0.01] paramMap["method"] = ["vsNon"] paramMap["stat"] = "tt" if os.path.exists("mut.cfg"): f = open("mut.cfg", "r") for line in f: if line.isspace(): continue pline = re.split("\t", line.rstrip("\n")) if line.startswith("distanceParams"): paramMap["dist"] = [int(i) for i in re.split(",", pline[1])] elif line.startswith("tstatParams"): paramMap["thresh"] = [float(i) for i in re.split(",", pline[1])] elif line.startswith("incrParams"): paramMap["inc"] = [float(i) for i in re.split(",", pline[1])] elif line.startswith("methodParams"): paramMap["method"] = [i for i in re.split(",", pline[1])] elif line.startswith("signalMethod"): paramMap["stat"] = int(pline[1]) elif line.startswith("cohortName"): paramMap["cohortName"] = pline[1] f.close() if "cohortName" not in paramMap: paramMap["cohortName"] = re.split("/", os.getcwd())[-1] ## foldMap rRepeats = 1 mFolds = 5 foldMap = {} if os.path.exists("fold.map"): (mapData, mapSamples, mapRepeats) = rCRSData("fold.map", retFeatures = True) for r, row in enumerate(mapRepeats): foldMap[r+1] = {} for col in mapSamples: mFolds = max(mFolds, int(mapData[col][row])) for m in range(mFolds): foldMap[r+1][m+1] = [] for col in mapSamples: foldMap[r+1][int(mapData[col][row])].append(col) if (r+1) == rRepeats: break else: for r in range(1, rRepeats+1): foldMap[r] = {} for m in range(1, mFolds+1): foldMap[r][m] = None ## check files pathwayFile = None cnvFile = None expFile = None dataMap = {} assert os.path.exists("%s/clusterFiles" % (paradigmDir)) for file in os.listdir("%s/clusterFiles" % (paradigmDir)): if file.endswith("pathway.tab"): pathwayFile = "%s/clusterFiles/%s" % (paradigmDir, file) elif file.endswith("CNV.tab"): cnvFile = "%s/clusterFiles/%s" % (paradigmDir, file) dataMap["cnv"] = cnvFile elif (file.endswith("Expression.tab") | file.endswith("Expression.vCohort.tab") | file.endswith("Expression.vNormal.tab")): expFile = "%s/clusterFiles/%s" % (paradigmDir, file) dataMap["exp"] = expFile assert (pathwayFile != None) assert (cnvFile != None) assert (expFile != None) paradigmFile = None if os.path.exists("%s/merge_merged_unfiltered.tab" % (paradigmDir)): paradigmFile = "%s/merge_merged_unfiltered.tab" % (paradigmDir) elif os.path.exists("%s/merge_merged.tab" % (paradigmDir)): paradigmFile = "%s/merge_merged.tab" % (paradigmDir) ## store feature, sample and pathway information dataFeatures = list(set(retColumns(cnvFile)) & set(retColumns(expFile))) includeFeatures = None if len(featureFile) != 0: includeFeatures = rList(featureFile) dataSamples = list(set(retRows(cnvFile)) & set(retRows(expFile))) if len(sampleFile) != 0: dataSamples = list(set(dataSamples) & set(rList(sampleFile))) (gNodes, gInteractions) = rPathway(pathwayFile) gfPathway = flattenPathway(Pathway(gNodes, gInteractions)) if not useFlattened: gPathway = Pathway(gNodes, gInteractions) else: gPathway = gfPathway mutationOrder = [] mutationMap = {} f = open(mutFile, "r") for line in f: if line.isspace(): continue pline = re.split("\t", line.rstrip("\r\n")) mutatedGene = pline[0] mutatedSamples = list(set(re.split(",", pline[2])) & set(dataSamples)) if mutatedGene in gPathway.nodes: if len(mutatedSamples) >= mutationThreshold: mutationMap[mutatedGene] = deepcopy(mutatedSamples) if includeFeatures is None: mutationOrder.append(mutatedGene) f.close() if includeFeatures is not None: for mutatedGene in includeFeatures: if mutatedGene in mutationMap: mutationOrder.append(mutatedGene) submitMap = {} for mutatedGene in mutationOrder: submitMap[mutatedGene] = deepcopy(mutationMap[mutatedGene]) if len(submitMap.keys()) >= maxFeatures: break ## run logger.info("options: " + str(options)) logger.info("starting make") writeScripts() s = Stack(branchGenes(dataSamples, dataFeatures, dataMap, submitMap, gPathway, paradigmDir, paramMap, foldMap, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def wrapParadigm(): ## parse arguments parser = OptionParser(usage = "%prog [options] attachment file:path [attachment file:path ...]") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-d", "--dogma", dest="dogmaPath", help="Path to PARADIGM Dogma Specification", default="") parser.add_option("-p", "--pathway", dest="pathwayPath", help="Path to PARADIGM Pathway Specification", default="") parser.add_option("-b", "--boundaries", dest="discBound", help="Data Discretization Bounds", default="") parser.add_option("-n", "--nulls", dest="nullBatches", help="Number of Null Samples", default="5") parser.add_option("-t", "--storedparam", dest="paramFile", help="Initial Parameter Starting Point", default="") parser.add_option("-s", "--skipem", action="store_false", dest="runEM", help="Skip Running EM", default=True) options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" evidList = [] for element in args: if element.startswith("rankAllFile"): evidList.append(re.sub("rankAllFile", "file", element)) else: evidList.append(element) if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) if len(options.discBound) == 0: disc = "0.3333;0.6667" else: disc = options.discBound if len(options.dogmaPath) == 0: dogma = "%s/%s" % (dogmaDir, dogmaDefault) else: dogma = options.dogmaPath if len(options.pathwayPath) == 0: pathway = "%s/%s" % (pathwayDir, pathwayDefault) else: pathway = options.pathwayPath nullBatches = int(options.nullBatches) if len(options.paramFile) == 0: paramFile = None else: paramFile = options.paramFile runEM = options.runEM logger.info("options: " + str(options)) ## run logger.info("starting prepare") s = Stack(prepareParadigm(" ".join(evidList), disc, paramFile, nullBatches, paradigmExec, inferSpec, dogma, pathway, runEM, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def main(): ## parse arguments parser = OptionParser(usage = "%prog [options] network IPL-matrix features") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-w", "--workdir", dest="workdir", help="Common Work directory", default="./") parser.add_option("-i", "--ipl", dest="iplFile", default = None) parser.add_option("-p", "--pathway", dest="pathwayZip", default=None) parser.add_option("-c", "--phenotype", dest="phenotypeFile", default=None) parser.add_option("-o", "--oz", dest="outputZip", default=None) parser.add_option("-s", "--score", dest="scoreFile", default=None) parser.add_option("-f", "--filter", dest="filterParams", default="0.0;0.0") parser.add_option("-b", "--background", dest="nBackground", default="0") options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" ## clean if len(args) == 1: if args[0] == "clean": print "rm -rf real* null* OCCAM__* LAYOUT background.R .jobTree" system("rm -rf real* null* OCCAM__* LAYOUT background.R .jobTree") sys.exit(0) ## parse arguments assert ((len(args) == 0) or (len(args) == 2) or (len(args) == 3)) if len(args) == 0: pathwayZip = options.pathwayZip if options.pathwayZip is not None else basepathway pathwayLib = os.path.join(options.workdir, "pathway") system("unzip %s -d %s" % (pathwayZip, pathwayLib)) paradigmPathway = None for file in os.listdir(pathwayLib): if file.endswith("_pathway.tab"): paradigmPathway = "%s/%s" % (pathwayLib, file) break scoreFile = None phenotypeFile = options.phenotypeFile dataFile = options.iplFile sampleList = [] for sample in retColumns(dataFile): if not sample.startswith("na_iter"): sampleList.append(sample) filterParams = options.filterParams nNulls = int(options.nBackground) outputZip = options.outputZip assert(os.path.exists(paradigmPathway)) assert(os.path.exists(phenotypeFile)) assert(os.path.exists(dataFile)) elif len(args) == 2: paradigmPathway = args[0] scoreFile = args[1] phenotypeFile = None dataFile = None sampleList = None filterParams = options.filterParams nNulls = 0 outputZip = options.outputZip assert(os.path.exists(paradigmPathway)) assert(os.path.exists(scoreFile)) elif len(args) == 3: paradigmPathway = args[0] scoreFile = None phenotypeFile = args[2] dataFile = args[1] sampleList = [] for sample in retColumns(dataFile): if not sample.startswith("na_iter"): sampleList.append(sample) filterParams = options.filterParams nNulls = int(options.nBackground) outputZip = options.outputZip assert(os.path.exists(paradigmPathway)) assert(os.path.exists(phenotypeFile)) assert(os.path.exists(dataFile)) ## run logger.info("options: " + str(options)) logger.info("starting make") writeScripts() s = Stack(prepareOCCAM(paradigmPathway, scoreFile, phenotypeFile, None, dataFile, sampleList, filterParams, nNulls, outputZip, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def main(): ## configure logger formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s') file_handler = logging.FileHandler('signature.log') file_handler.setFormatter(formatter) file_handler.setLevel(logging.INFO) stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) stream_handler.setLevel(logging.ERROR) logger.addHandler(file_handler) logger.addHandler(stream_handler) ## check for fresh run if os.path.exists(".jobTree"): logger.warning( "WARNING: '.jobTree' directory found, remove it first to start a fresh run\n" ) ## parse arguments parser = OptionParser(usage="%prog [options] data_matrix phenotype_matrix") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help="Add as child of jobFile rather than new jobTree") parser.add_option( "-b", "--bootstrap", dest="bootstrap_size", default=0, help="Number of bootstrap samples to estimate subnetwork robustness") parser.add_option( "-n", "--null", dest="null_size", default=0, help="Number of null samples to estimate subnetwork significance") parser.add_option("-p", "--permute", dest="null_permute", default="paradigm", help="Permutation method for generation of null samples") parser.add_option("-m", "--method", dest="signature_method", default="sam", help="Differential method for computing signatures") parser.add_option( "-z", "--seed", dest="seed", default=None, help="Random seed used for bootstrap and null generation") options, args = parser.parse_args() logger.info("options: %s" % (str(options))) print "Using Batch System '%s'" % (options.batchSystem) if len(args) != 2: logger.error("ERROR: incorrect number of arguments\n") sys.exit(1) data_file = os.path.abspath(args[0]) phenotype_file = os.path.abspath(args[1]) ## set parameters parameters = Parameters(signature_method=options.signature_method, bootstrap_size=int(options.bootstrap_size), null_size=int(options.null_size), null_method=options.null_permute, random_seed=options.seed) ## run s = Stack( branchSignatures(data_file, phenotype_file, parameters, os.getcwd().rstrip("/"))) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" jobtree_dir = options.jobTree.rstrip("/") lasttree_dir = jobtree_dir + "_previous" failed = s.startJobTree(options) if failed: logger.warning("WARNING: %d jobs failed" % (failed)) else: logger.info("Run complete!") if os.path.exists(lasttree_dir): shutil.rmtree(lasttree_dir) if os.path.exists(jobtree_dir): shutil.move(jobtree_dir, lasttree_dir)
def as_main(): ## check for fresh run if os.path.exists('.jobTree'): logger( 'WARNING: .jobTree directory found, remove it first to start a fresh run\n' ) ## parse arguments parser = OptionParser( usage='%prog [options] paradigm_directory model_directory') Stack.addJobTreeOptions(parser) parser.add_option('--jobFile', help='Add as child of jobFile rather than new jobTree') parser.add_option('-n', '--nulls', dest='nulls', default=0) parser.add_option('-b', '--batchsize', dest='batch_size', default=50) parser.add_option('-y', '--public', action='store_true', dest='paradigm_public', default=False) parser.add_option('-g', '--galaxy', dest='galaxy_run', action='store_true', default=False) options, args = parser.parse_args() logger('Using Batch System : %s\n' % (options.batchSystem)) if len(args) == 1: if args[0] == 'clean': command = 'rm -rf .jobTree analysis' logger(command) os.system(command) sys.exit(0) assert (len(args) == 2) paradigm_directory = os.path.abspath(args[0]) model_directory = os.path.abspath(args[1]) ## set Galaxy Executables global paradigm_executable, circleplot_executable if options.galaxy_run: paradigm_executable = os.path.join(base_directory, 'paradigm') ## set Paradigm files paradigm_setup = ParadigmSetup(paradigm_directory, nulls=int(options.nulls), batch_size=int(options.batch_size), public=options.paradigm_public) ## gather models model_list = [] for file in os.listdir(model_directory): if os.path.exists('%s/%s/upstream_pathway.tab' % (model_directory, file)) and os.path.exists( '%s/%s/downstream_pathway.tab' % (model_directory, file)): model = Model('%s/%s' % (model_directory, file)) model.setSeed() model_list.append(model) ## run s = Stack(queueModels(model_list, paradigm_setup, os.getcwd().rstrip('/'))) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = './.jobTree' failed = s.startJobTree(options) #failed = '' #queueModels(model_list, paradigm_setup, os.getcwd().rstrip('/')).run() if failed: logger('%d jobs failed\n' % failed) else: os.system('rm -rf .lastjobTree') os.system('mv .jobTree .lastjobTree')
def wrapParadigm(): ## parse arguments parser = OptionParser( usage="%prog [options] attachment file:path [attachment file:path ...]" ) Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help="Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-d", "--dogma", dest="dogmaPath", help="Path to PARADIGM Dogma Specification", default="") parser.add_option("-p", "--pathway", dest="pathwayPath", help="Path to PARADIGM Pathway Specification", default="") parser.add_option("-b", "--boundaries", dest="discBound", help="Data Discretization Bounds", default="") parser.add_option("-n", "--nulls", dest="nullBatches", help="Number of Null Samples", default="5") parser.add_option("-t", "--storedparam", dest="paramFile", help="Initial Parameter Starting Point", default="") parser.add_option("-s", "--skipem", action="store_false", dest="runEM", help="Skip Running EM", default=True) options, args = parser.parse_args() print "Using Batch System '" + options.batchSystem + "'" evidList = [] for element in args: if element.startswith("rankAllFile"): evidList.append(re.sub("rankAllFile", "file", element)) else: evidList.append(element) if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) if len(options.discBound) == 0: disc = "0.3333;0.6667" else: disc = options.discBound if len(options.dogmaPath) == 0: dogma = "%s/%s" % (dogmaDir, dogmaDefault) else: dogma = options.dogmaPath if len(options.pathwayPath) == 0: pathway = "%s/%s" % (pathwayDir, pathwayDefault) else: pathway = options.pathwayPath nullBatches = int(options.nullBatches) if len(options.paramFile) == 0: paramFile = None else: paramFile = options.paramFile runEM = options.runEM logger.info("options: " + str(options)) ## run logger.info("starting prepare") s = Stack( prepareParadigm(" ".join(evidList), disc, paramFile, nullBatches, paradigmExec, inferSpec, dogma, pathway, runEM, os.getcwd())) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print("%d jobs failed" % failed) else: logger.info("Run complete!") system("rm -rf .lastjobTree") system("mv .jobTree .lastjobTree")
def wrapParadigm(): ## parse arguments parser = OptionParser(usage = "%prog [options] attachment file:path [attachment file:path ...]") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as a child of jobFile rather " + "than making a new jobTree") parser.add_option("-w", "--workdir", dest="workdir", help="Common Work directory", default="./") parser.add_option("-n", "--nulls", dest="nulls", help="Number of Null Samples", default="5") parser.add_option("-d", "--dogma", dest="dogma", help="Path to PARADIGM Dogma Specification", default=basedogma) parser.add_option("-i", "--imap", dest="imap", help="Path to PARADIGM Interaction Map Specification", default=baseimap) parser.add_option("-t", "--storedparam", dest="param", help="Initial Parameter Starting Point", default=baseparams) parser.add_option("--xgmml", dest="xgmml", help="Path to PARADIGM Pathway XGMML file", default=None) parser.add_option("-p", "--pathway", dest="pathway", help="Path to PARADIGM Pathway Specification", default=basepathway) parser.add_option("-b", "--boundaries", dest="disc", help="Data Discretization Bounds", default="0.33;0.67") parser.add_option("-s", "--skipem", action="store_false", dest="em", help="Skip Running EM", default=True) parser.add_option("--lb-max", dest="lb_max", help="Loopy Belief Max iterations", default=10000) parser.add_option("-o", "--output", dest="output_paradigm", help="Unfiltered Output", default="paradigm.output") parser.add_option("--op", "--output-params", dest="output_params", help="Parameter Output", default=None) parser.add_option("--oc", "--output-config", dest="output_config", help="Config Output", default=None) parser.add_option("--of", "--output-files", dest="output_files", help="Output Files", default=None) parser.add_option("-z", dest="private_paradigm", help="This is such bullshit", action="store_true", default=False) options, args = parser.parse_args() logging.info("options: " + str(options)) print "Using Batch System '" + options.batchSystem + "'" evidList = [] for i, element in enumerate(args): if i % 2 == 1: (fileType, filePath) = args[i].split(":") evidList.append("%s:%s" % (fileType, os.path.abspath(filePath))) else: evidList.append(args[i]) if (len(evidList) % 2 == 1) | (len(evidList) == 0): sys.stderr.write("ERROR: incorrect number of arguments\n") sys.exit(1) workdir = os.path.abspath(options.workdir) if not os.path.exists(workdir): os.makedirs(workdir) nullBatches = int(options.nulls) dogma = os.path.abspath(options.dogma) imap = os.path.abspath(options.imap) params = os.path.abspath(options.param) disc=options.disc paramFile=os.path.abspath(options.param) runEM = options.em if not os.path.exists(workdir): os.makedirs(workdir) dogmaLib = os.path.join(workdir, "dogma") pathwayLib = os.path.join(workdir, "pathway") os.makedirs(dogmaLib) os.makedirs(pathwayLib) shutil.copy(dogma, dogmaLib) shutil.copy(imap, dogmaLib) if options.xgmml: pathway = os.path.join(pathwayLib, "pid_tmp_pathway.tab") system("%s %s %s %s" % (sys.executable, xgmmlExec, options.xgmml, pathway)) else: pathway = os.path.abspath(options.pathway) shutil.copy(pathway, os.path.join(pathwayLib, "pid_tmp_pathway.tab")) ## run logging.info("starting prepare") argSpec = inferSpec % (options.lb_max) s = Stack(prepareParadigm(evidSpec=" ".join(evidList), disc=disc, paramFile=paramFile, nullBatches=nullBatches, paradigmExec=paradigmExec, inferSpec=argSpec, dogmaLib=dogmaLib, pathwayLib=pathwayLib, em=runEM, directory=workdir, private_paradigm=options.private_paradigm )) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" failed = s.startJobTree(options) if failed: print ("%d jobs failed" % failed) else: shutil.copy( os.path.join(options.workdir, "merge_merged_unfiltered.all.tab"), options.output_paradigm) if options.output_params is not None: shutil.copy( os.path.join(options.workdir, "params.txt"), options.output_params) if options.output_config is not None: shutil.copy( os.path.join(options.workdir, "config.txt"), options.output_config) if options.output_files is not None: system("zip -r outputFiles.zip outputFiles") shutil.copy( os.path.join(options.workdir, "outputFiles.zip"), options.output_files) logging.info("Run complete!") if os.path.exists(".lastjobTree"): system("rm -rf .lastjobTree") if os.path.exists(".jobTree"): system("mv .jobTree .lastjobTree")
def gp_main(): ## check for fresh run if os.path.exists(".jobTree"): logging.warning("WARNING: '.jobTree' directory found, remove it first to start a fresh run\n") ## parse arguments parser = OptionParser(usage = "%prog [options] attachment file:path [attachment file:path ...]") Stack.addJobTreeOptions(parser) parser.add_option("--jobFile", help = "Add as a child of jobFile rather than making a new jobTree") parser.add_option("-w", "--workdir", dest = "work_dir", default = "./", help = "Directory to perform work in") parser.add_option("-n", "--nulls", dest = "null_size", default="5", help = "Number of null samples to be generated per real sample") parser.add_option("-d", "--dogma", dest = "dogma_lib", default = standard_dogma, help = "Directory of PARADIGM Dogma specification") parser.add_option("-t", "--param", dest = "param_file", default = None, help = "Path to initial PARADIGM Parameters") parser.add_option("-p", "--pathway", dest = "pathway_lib", default=standard_pathway, help = "Path to PARADIGM Pathway (directory/file/zip)") parser.add_option("-b", "--boundaries", dest = "disc", default="0.33;0.67", help = "Data discretization boundaries") parser.add_option("-s", "--skipem", action = "store_false", dest="run_em", default=True, help = "Skip EM steps") parser.add_option("-y", dest = "paradigm_public", action = "store_true", default = False, help = "This flag must be enabled when using the publically available version of paradigm") parser.add_option("-o", "--output-ipls", dest = "output_ipls", default = "unfiltered.all.tab", help = "Unfiltered Output") parser.add_option("--op", "--output-params", dest = "output_params", default = None, help = "Parameter Output") parser.add_option("--oc", "--output-config", dest = "output_config", default = None, help = "Config Output") parser.add_option("--of", "--output-files", dest = "output_files", default = None, help = "Output Files") options, args = parser.parse_args() logging.info("options: %s" % (str(options))) print "Using Batch System '%s'" % (options.batchSystem) evidence_list = [] for i, element in enumerate(args): if i % 2 == 1: (file_type, file_path) = args[i].split(":") evidence_list.append("%s:%s" % (file_type, os.path.abspath(file_path))) else: evidence_list.append(args[i]) if (len(evidence_list) % 2 == 1) | (len(evidence_list) == 0): logging.error("ERROR: incorrect number of arguments\n") sys.exit(1) work_dir = os.path.abspath(options.work_dir) if not os.path.exists(work_dir): os.makedirs(work_dir) null_size = int(options.null_size) disc = options.disc if options.param_file is not None: param_file = os.path.abspath(options.param_file) else: param_file = None run_em = options.run_em ## import dogma and pathway libraries if zipfile.is_zipfile(options.dogma_lib): zf = zipfile.ZipFile(options.dogma_lib, "r") dogma_lib = os.path.join(work_dir, "dogma") zf.extractall(dogma_lib) zf.close() elif os.path.isdir(options.dogma_lib): dogma_lib = os.path.abspath(options.dogma_lib) else: logging.error("ERROR: dogma cannot be a regular file\n") sys.exit(1) if zipfile.is_zipfile(options.pathway_lib): zf = zipfile.ZipFile(options.pathway_lib, "r") pathway_lib = os.path.join(work_dir, "pathway") zf.extractall(pathway_lib) zf.close() elif os.path.isdir(options.pathway_lib): pathway_lib = os.path.abspath(options.pathway_lib) else: logging.error("ERROR: pathway cannot be a regular file\n") sys.exit(1) ## initialize the stack and run logging.info("starting prepare") s = Stack(PrepareParadigm(evidence_spec=" ".join(evidence_list), disc = disc, param_file = param_file, null_size = null_size, paradigm_exec = paradigm_exec, inference_spec = standard_inference, dogma_lib = dogma_lib, pathway_lib = pathway_lib, run_em = run_em, directory = work_dir, paradigm_public = options.paradigm_public)) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" jobtree_dir = options.jobTree.rstrip("/") lasttree_dir = jobtree_dir + "_previous" failed = s.startJobTree(options) if failed: logging.warning("WARNING: %d jobs failed" % (failed)) else: shutil.copy(os.path.join(options.work_dir, "merge_merged_unfiltered.all.tab"), options.output_ipls) if options.output_params is not None: shutil.copy(os.path.join(options.work_dir, "params.txt"), options.output_params) if options.output_config is not None: shutil.copy(os.path.join(options.work_dir, "config.txt"), options.output_config) if options.output_files is not None: zip_file = zipfile.ZipFile("outputFiles.zip", "w") zipDirectory("outputFiles", zip_file) zip_file.close() shutil.copy(os.path.join(options.work_dir, "outputFiles.zip"), options.output_files) logging.info("Run complete!") if os.path.exists(lasttree_dir): shutil.rmtree(lasttree_dir) if os.path.exists(jobtree_dir): shutil.move(jobtree_dir, lasttree_dir)
def gp_main(): ## check for fresh run if os.path.exists(".jobTree"): logging.warning( "WARNING: '.jobTree' directory found, remove it first to start a fresh run\n" ) ## parse arguments parser = OptionParser( usage="%prog [options] attachment file:path [attachment file:path ...]" ) Stack.addJobTreeOptions(parser) parser.add_option( "--jobFile", help="Add as a child of jobFile rather than making a new jobTree") parser.add_option("-w", "--workdir", dest="work_dir", default="./", help="Directory to perform work in") parser.add_option( "-n", "--nulls", dest="null_size", default="5", help="Number of null samples to be generated per real sample") parser.add_option("-d", "--dogma", dest="dogma_lib", default=standard_dogma, help="Directory of PARADIGM Dogma specification") parser.add_option("-t", "--param", dest="param_file", default=None, help="Path to initial PARADIGM Parameters") parser.add_option("-p", "--pathway", dest="pathway_lib", default=standard_pathway, help="Path to PARADIGM Pathway (directory/file/zip)") parser.add_option("-b", "--boundaries", dest="disc", default="0.33;0.67", help="Data discretization boundaries") parser.add_option("-s", "--skipem", action="store_false", dest="run_em", default=True, help="Skip EM steps") parser.add_option( "-y", dest="paradigm_public", action="store_true", default=False, help= "This flag must be enabled when using the publically available version of paradigm" ) parser.add_option("-o", "--output-ipls", dest="output_ipls", default="unfiltered.all.tab", help="Unfiltered Output") parser.add_option("--op", "--output-params", dest="output_params", default=None, help="Parameter Output") parser.add_option("--oc", "--output-config", dest="output_config", default=None, help="Config Output") parser.add_option("--of", "--output-files", dest="output_files", default=None, help="Output Files") options, args = parser.parse_args() logging.info("options: %s" % (str(options))) print "Using Batch System '%s'" % (options.batchSystem) evidence_list = [] for i, element in enumerate(args): if i % 2 == 1: (file_type, file_path) = args[i].split(":") evidence_list.append("%s:%s" % (file_type, os.path.abspath(file_path))) else: evidence_list.append(args[i]) if (len(evidence_list) % 2 == 1) | (len(evidence_list) == 0): logging.error("ERROR: incorrect number of arguments\n") sys.exit(1) work_dir = os.path.abspath(options.work_dir) if not os.path.exists(work_dir): os.makedirs(work_dir) null_size = int(options.null_size) disc = options.disc if options.param_file is not None: param_file = os.path.abspath(options.param_file) else: param_file = None run_em = options.run_em ## import dogma and pathway libraries if zipfile.is_zipfile(options.dogma_lib): zf = zipfile.ZipFile(options.dogma_lib, "r") dogma_lib = os.path.join(work_dir, "dogma") zf.extractall(dogma_lib) zf.close() elif os.path.isdir(options.dogma_lib): dogma_lib = os.path.abspath(options.dogma_lib) else: logging.error("ERROR: dogma cannot be a regular file\n") sys.exit(1) if zipfile.is_zipfile(options.pathway_lib): zf = zipfile.ZipFile(options.pathway_lib, "r") pathway_lib = os.path.join(work_dir, "pathway") zf.extractall(pathway_lib) zf.close() elif os.path.isdir(options.pathway_lib): pathway_lib = os.path.abspath(options.pathway_lib) else: logging.error("ERROR: pathway cannot be a regular file\n") sys.exit(1) ## initialize the stack and run logging.info("starting prepare") s = Stack( PrepareParadigm(evidence_spec=" ".join(evidence_list), disc=disc, param_file=param_file, null_size=null_size, paradigm_exec=paradigm_exec, inference_spec=standard_inference, dogma_lib=dogma_lib, pathway_lib=pathway_lib, run_em=run_em, directory=work_dir, paradigm_public=options.paradigm_public)) if options.jobFile: s.addToJobFile(options.jobFile) else: if options.jobTree == None: options.jobTree = "./.jobTree" jobtree_dir = options.jobTree.rstrip("/") lasttree_dir = jobtree_dir + "_previous" failed = s.startJobTree(options) if failed: logging.warning("WARNING: %d jobs failed" % (failed)) else: shutil.copy( os.path.join(options.work_dir, "merge_merged_unfiltered.all.tab"), options.output_ipls) if options.output_params is not None: shutil.copy(os.path.join(options.work_dir, "params.txt"), options.output_params) if options.output_config is not None: shutil.copy(os.path.join(options.work_dir, "config.txt"), options.output_config) if options.output_files is not None: zip_file = zipfile.ZipFile("outputFiles.zip", "w") zipDirectory("outputFiles", zip_file) zip_file.close() shutil.copy(os.path.join(options.work_dir, "outputFiles.zip"), options.output_files) logging.info("Run complete!") if os.path.exists(lasttree_dir): shutil.rmtree(lasttree_dir) if os.path.exists(jobtree_dir): shutil.move(jobtree_dir, lasttree_dir)