import random as random_mod import sys, os, pickle __parallel_loaded__ = False _cpu_count = 1 try: import multiprocessing as processing __parallel_loaded__ = True _cpu_count = processing.cpu_count except: try: import processing __parallel_loaded__ = True _cpu_count = processing.cpuCount() except: #print 'processing not installed no parallel processing possible' pass import model from lib.Simplex import Simplex # Add current path to the system paths # just in case some user make a directory change sys.path.append(os.getcwd()) #============================================================================== # class: DiffEv
queue.task_done() if __name__ == '__main__': start = time.time() src_path = "/media/tx-eva-data/NAS/基础数据库/华中科技大学同济医学院附属同济医院/2017_07_13_abnormal" dec_path = "/media/tx-eva-data/NAS/原始数据库/华中科技大学同济医学院附属同济医院" queue = multiprocessing.JoinableQueue() list_p = [] for i in range(multiprocessing.cpu_count() - 2): process = multiprocessing.Process(target=Worker, args=(queue, )) process.daemon = True process.start() list_p.append(process) for dirpath, dirnames, filenames in os.walk(src_path): for file in filenames: dcm_path = os.path.join(dirpath, file) queue.put([dcm_path, dec_path]) queue.join() for i in range(multiprocessing.cpuCount() - 2): queue.put(None) for p in list_p: p.join() elapsed = time.time() - start print('main end! %f' % elapsed)
# USER-SPECIFIC PATHS AND VALUES # We'll be using SWarp. Define the proper bin executable here. This is # particularly important if you have different installations and want to use a # specific version in the reduction. Also, note that the SWarp packaged with # Scisoft probably won't work properly on Intel 64-bit Macs. It is strongly # advised that you recompile and install SWarp from source if SWarp's coadd # mosaics are null images. swarp_bin = "/Applications/scisoft//i386/bin/swarp" # We'll also be using some WCSTools. There should be no problem with the Scisoft # version of thses. sethead_bin = "/Applications/scisoft//i386/bin/sethead" # For parallel processing we need the number of available CPUs. You could hard- # code this value to something else, but it would result in non-optimal # performance. if doparallel == 1: numprocessors = cpuCount() else: numprocessors = 1 # Debugging mode outputs intermediate image files to help track down problems. # It requires more time to run because of the additional hard disk reads/writes. DEBUG = False #------------------------------------------------------------------------------- # FUNCTION DEFINITIONS # Simple function allowing parallelization of mosaicing with SWarp. def run_swarp(command): system(command) return
# advised that you recompile and install SWarp from source if SWarp's coadd # mosaics are null images. swarp_bin = "swarp" # We'll also be using some WCSTools. There should be no problem with the Scisoft # version of thses. sethead_bin = "sethead" # For parallel processing we need the number of available CPUs. You could hard- # code this value to something else, but it would result in non-optimal # performance. # AMORGAN ADDS for WCS FITTING Change if your python version is different. # Will want to avoid having to load this in future versions. pypath = "" if doparallel == 1: numprocessors = cpuCount() else: numprocessors = 1 # Debugging mode outputs intermediate image files to help track down problems. # It requires more time to run because of the additional hard disk reads/writes. DEBUG = False #------------------------------------------------------------------------------- # FUNCTION DEFINITIONS # Simple function allowing parallelization of mosaicing with SWarp. def run_swarp(command): system(command) return #------------------------------------------------------------------------------- # BEGIN MAIN PROGRAM if not os.environ.has_key("Q_DIR"):
import version __version__ = version.version def complement(s, _comp=string.maketrans('ATCG', 'TAGC')): return s.translate(_comp) def revcomp(s, _comp=string.maketrans('ATCG', 'TAGC')): return s.translate(_comp)[::-1] CPU_COUNT = 4 try: import multiprocessing as processing CPU_COUNT = processing.cpu_count() except ImportError: import processing CPU_COUNT = processing.cpuCount() def write_c2t(fasta_name, unconverted, colorspace=False): """ given a fasta file, write a new file: `some.fr.c2t.fasta` which contains: + the same headers prefixed with 'f' with all C's converted to T + headers prefixed with 'r' reverse complemented with all C's converted to T. if unconverted is false, then also save a file with the forward and reverse without conversion. """ d = op.join(op.dirname(fasta_name), "bowtie_index") if colorspace: d += "_colorspace" if not op.exists(d): os.mkdir(d)
import test # from test import pystone import multiprocessing import time STONES_PER_PROCESS = 10 * test.pystone.LOOPS def f(q): t = test.pystone.pystones(STONES_PER_PROCESS) q.put(t, block=True) if __name__ == '__main__': print 'multiprocessing test.pystones() benchmark' print 'You have ' + str(multiprocessing.cpuCount()) + ' CPU(s)' print 'Processes\tPystones\tWall time\tpystones/s' results = multiprocessing.Queue() for N in range(1, multiprocessing.cpuCount() + 3): p = [] q = multiprocessing.Queue() results = [] for m in range(1, N + 1): p.append(multiprocessing.Process(target=f, args=(q, ))) start = time.time() for pr in p: pr.start() for r in p:
def complement(s, _comp=string.maketrans('ATCG', 'TAGC')): return s.translate(_comp) def revcomp(s, _comp=string.maketrans('ATCG', 'TAGC')): return s.translate(_comp)[::-1] CPU_COUNT = 4 try: import multiprocessing as processing CPU_COUNT = processing.cpu_count() except ImportError: import processing CPU_COUNT = processing.cpuCount() def write_c2t(fasta_name, unconverted, colorspace=False): """ given a fasta file, write a new file: `some.fr.c2t.fasta` which contains: + the same headers prefixed with 'f' with all C's converted to T + headers prefixed with 'r' reverse complemented with all C's converted to T. if unconverted is false, then also save a file with the forward and reverse without conversion. """ d = op.join(op.dirname(fasta_name), "bowtie_index") if colorspace: d += "_colorspace"