Beispiel #1
0
def entry_point():
    args, metric = parser.parse_args()

    if hasattr(args, 'sclarans_parallel') and args.sclarans_parallel == 'dtm':
        from deap import dtm
        dtm.start(main, args)
    else:
        main(args, metric)
Beispiel #2
0
def entry_point():
    args, metric = parser.parse_args()
    
    if hasattr(args, 'sclarans_parallel')  and args.sclarans_parallel == 'dtm':
        from deap import dtm
        dtm.start(main, args)
    else:
        main(args, metric)
            outf.write("%s\n" % (tree[2]))
        outf.close()
    # compute bootreps on genetrees from above
    if args.run == 'both':
        sys.stdout.write("Running boostraps of genetrees...\n")
        # get models for each locus based on genetrees in-memory
        models = dict([[tree[0], tree[1]] for tree in genetrees])
        boostrap_all_loci(args, models, alns)
    # compute bootreps on genetrees from a file
    if args.run == 'bootstraps':
        sys.stdout.write("Running boostraps...\n")
        # get models for each locus based on genetrees in genetree file
        models = get_models_from_genetrees(args.genetrees)
        boostrap_all_loci(args, models, alns)


if __name__ == '__main__':
    args = get_args()
    if args.parallelism == 'mpi':
        from deap.dtm import map as mmap
        from deap.dtm import start
        start(main)
    elif args.parallelism == 'multiprocessing':
        from multiprocessing import Pool
        pool = Pool(args.cores)
        mmap = pool.map
        main()
    elif args.parallelism == 'single':
        mmap = map
        main()
Beispiel #4
0
#    GNU Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with DEAP. If not, see <http://www.gnu.org/licenses/>.

"""
A very simple example of recursive nested tasks.
Each task maps 2 others tasks, each of these 2 tasks maps 2 others, etc.,
up to RECURSIVITY_LEVEL.

Setting RECURSIVITY_LEVEL to a high value (typically more than 16) may
produce some strange OS errors (as the kernel will probably complain about
the number of active threads...
"""

from deap import dtm

RECURSIVITY_LEVEL = 12

def recursiveFunc(level):
    if level == 1:
        return 1
    else:
        args = [level-1] * 2
        s = sum(dtm.map(recursiveFunc, args))
        if level == RECURSIVITY_LEVEL:
            print("2^"+str(level)+" = " + str(s))
        return s

dtm.start(recursiveFunc, RECURSIVITY_LEVEL)
Beispiel #5
0
#    License along with DEAP. If not, see <http://www.gnu.org/licenses/>.

"""
Distributed computation of the prime numbers below 45,000 using DTM.
"""

import time
import math

from deap import dtm

def primaryTest(nbr):
    if nbr <= 2:
        return True
    
    for i in range(2, int(math.sqrt(nbr))+1):
        if nbr % i == 0:
            return False
    return True

def main(upTo):
    beginTime = time.time()
    listNbr = range(3,upTo,2)
    print("Computation begin...")
    listPrimes = dtm.filter(primaryTest, listNbr)
    
    print("Found " + str(len(listPrimes)) + " prime numbers between 3 and " + str(upTo))
    print("Computation time : " + str(time.time()-beginTime))

dtm.start(main, 45003)
            outf.write("%s\n" % (tree[2]))
        outf.close()
    # compute bootreps on genetrees from above
    if args.run == 'both':
        sys.stdout.write("Running bootstraps of genetrees...\n")
        # get models for each locus based on genetrees in-memory
        models = dict([[tree[0], tree[1]] for tree in genetrees])
        boostrap_all_loci(args, models, alns)
    # compute bootreps on genetrees from a file
    if args.run == 'bootstraps':
        sys.stdout.write("Running boostraps...\n")
        # get models for each locus based on genetrees in genetree file
        models = get_models_from_genetrees(args.genetrees)
        boostrap_all_loci(args, models, alns)


if __name__ == '__main__':
    args = get_args()
    if args.parallelism == 'mpi':
        from deap.dtm import map as mmap
        from deap.dtm import start
        start(main)
    elif args.parallelism == 'multiprocessing':
        from multiprocessing import Pool
        pool = Pool(args.cores)
        mmap = pool.map
        main()
    elif args.parallelism == 'single':
        mmap = map
        main()
Beispiel #7
0
    parser.add_option("--job_name", dest="job_name", default="fold", help="name for submitted job.  default:%default", metavar="NAME")
    parser.add_option("--processors", dest="np", default=4, help="number of processors to use.  default:%default", type="int", metavar="NP")
    parser.add_option("--notify", dest="notify", default=None, help="email address to notify of start, errors and completion", metavar="EMAIL")
    parser.add_option("--foldparams", dest="fp", type="str", action="store", default="-W_80_-L_40_-u_1", help="folding parameters default=%default", metavar="PARAMETER STRING")
    parser.add_option("--start", dest="start", default=False, action="store_true", help=SUPPRESS_HELP) #private, don't use

    (options, args) = parser.parse_args()
    options.prefix = os.getcwd()

    #main(options)
    #exit()


    
    if options.start is True:
        dtm.start(main, options)
        #main(options)
    
    else:
        scriptName = os.path.join(options.prefix, options.job_name+".RUNFOLD.sh")
        runerr = os.path.join(options.prefix, options.job_name+ ".err")
        runout = os.path.join(options.prefix, options.job_name+ ".out")
        shScript = open(scriptName, 'w')
        host = Popen(["hostname"], stdout=PIPE).communicate()[0].strip()

        if "optiputer" in host or "compute" in host:
            shScript.write("#!/bin/bash\n#$ -N %s\n#$ -S /bin/bash\n#$ -V\n#$ -pe mpi %d\n#$ -cwd\n#$ -o %s\n#$ -e %s\n#$ -l bigmem\n" %(options.job_name, options.np, runout, runerr))
            if options.notify is not None:
                shScript.write("#$ -notify\n#$ -m abe\n#$ -M %s\n" %(options.notify))

            shScript.write("/opt/openmpi/bin/mpirun -np $NSLOTS -machinefile $TMPDIR/machines python %s --start\n" %(" ".join(sys.argv)))
    ######################################################################################
    
    _logger.info("[%s] Testing parameters and exceptions handling...", time.time()-beginTime)
    
    applyParamPassr = dtm.apply(applyFunc4, 1, 2, "abc", bb={'a':2, 'b':3, 'c':4}, cc=range(10), dd=13.37)
    applyParamPasst = applyFunc4(1, 2, "abc", bb={'a':2, 'b':3, 'c':4}, cc=range(10), dd=13.37)
    
    if applyParamPassr == applyParamPasst:
        _logger.info("[%s] DTM parameters passing test successful", time.time()-beginTime)
    else:
        _logger.warning("[%s] DTM parameters passing test FAILED", time.time()-beginTime)
        countFailed += 1
        
    
    try:
        applyExceptTestr = dtm.map(applyFunc3, [-2,-1,0])
    except ZeroDivisionError:
        _logger.info("[%s] DTM exception catch test successful", time.time()-beginTime)
    else:
        _logger.warning("[%s] DTM exception catch test FAILED", time.time()-beginTime)
        countFailed += 1
        
    
    _logger.info("[%s] DTM test suite done with %i errors", time.time()-beginTime, countFailed)
    
    return 0

#dtm.setOptions(setTraceMode=True)
dtm.start(main)

Beispiel #9
0
could stride a little at the begining, but its not recommended.""")
        sys.exit(1)

    trajs = load_trajectories(args.project, args.stride)
    logger.info('Loaded %d trajs', len(trajs))

    clusterer = cluster(metric, trajs, args)

    if not isinstance(clusterer, clustering.Hierarchical):
        generators = clusterer.get_generators_as_traj()
        logger.info('Saving %s', args.generators)
        generators.save_to_lhdf(args.generators)
        if args.stride == 1:
            assignments = clusterer.get_assignments()
            distances = clusterer.get_distances()

            logger.info('Since stride=1, Saving %s', args.assignments)
            logger.info('Since stride=1, Saving %s', args.distances)
            io.saveh(args.assignments, assignments)
            io.saveh(args.distances, distances)


if __name__ == '__main__':
    args, metric = parser.parse_args()

    if hasattr(args, 'sclarans_parallel') and args.sclarans_parallel == 'dtm':
        from deap import dtm
        dtm.start(main, args)
    else:
        main(args, metric)
Beispiel #10
0
#    the License, or (at your option) any later version.
#
#    DEAP is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with DEAP. If not, see <http://www.gnu.org/licenses/>.

"""
Calculation of Pi using a Monte Carlo method.
"""

from math import hypot
from random import random

from deap import dtm

def test(tries):
    return sum(hypot(random(), random()) < 1 for i in range(tries))

def calcPi(n, t):
    expr = dtm.repeat(test, n, t)
    pi_value = 4. * sum(expr) / float(n*t)
    print("pi = " + str(pi_value))
    return pi_value

dtm.setOptions(setTraceMode=True, taskGranularity=0.01)
dataPi = dtm.start(calcPi, 3000, 5000)
Beispiel #11
0
    _logger.info("Lauching %s on worker %s", taskCmd, dtm.getWorkerId())
    return subprocess.call(taskCmd, stdout=sys.stdout, stderr=sys.stderr)

def main(fTasks):
    tasksList = []
    with open(fTasks) as f:
        for line in f:
            line = line.strip()
            if len(line) > 0:
                tasksList.append(line)
    
    retVals = dtm.map(executor, tasksList)
    if sum(retVals) > 0:
        _logger.warning("Some tasks did not run well (return code > 0) :")
        for i in range(len(retVals)):
            if retVals[i] > 0:
                _logger.warning("%s returns %s", tasksList[i], retVals[i])
    
    _logger.info("Done")

if __name__ == '__main__':
    cmdLine = optparse.OptionParser(description = "Simple script scheduler (sssched) for remote job management")
    cmdLine.add_option("-c", "--commands", 
                        dest = "commandsFileName",
                        action = "store",
                        help = "File with list of commands to perform (default:\"commands.lst\")",
                        type = "string")
    
    (options, args) = cmdLine.parse_args()
    dtm.start(main, options.commandsFileName)
Beispiel #12
0
# -*- coding: utf-8 -*-
from Tree import *
import sys
import time
from deap import dtm

def main():
    return executeTree()
    
importTree(sys.argv[1] if len(sys.argv) > 1 else "tree.txt")
registerMap(dtm.map)

if __name__=="__main__":
    bt = time.time()
    dtm.start(main)
    totalTime = time.time() - bt
    print("total time : {}".format(totalTime))
Beispiel #13
0
    clusterer = cluster(metric, trajs, args)
    
    if not isinstance(clusterer, clustering.Hierarchical):
        generators = clusterer.get_generators_as_traj()
        logger.info('Saving %s', args.generators)
        generators.save_to_lhdf(args.generators)
        if args.stride == 1:
            assignments = clusterer.get_assignments()
            distances = clusterer.get_distances()
            
            logger.info('Since stride=1, Saving %s', args.assignments)
            logger.info('Since stride=1, Saving %s', args.distances)
            io.saveh(args.assignments, assignments)
            io.saveh(args.distances, distances)

if __name__ == '__main__':
    args, metric = parser.parse_args()
    
    if hasattr(args, 'sclarans_parallel')  and args.sclarans_parallel == 'dtm':
        from deap import dtm
        dtm.start(main, args)
    else:
        main(args, metric)