import random as random_mod
import sys, os, pickle
__parallel_loaded__ = False
_cpu_count = 1
try:
import multiprocessing as processing
__parallel_loaded__ = True
_cpu_count = processing.cpu_count
except:
try:
import processing
__parallel_loaded__ = True
_cpu_count = processing.cpuCount()
except:
#print 'processing not installed no parallel processing possible'
pass
import model
from lib.Simplex import Simplex
# Add current path to the system paths
# just in case some user make a directory change
sys.path.append(os.getcwd())
#==============================================================================
# class: DiffEv
queue.task_done()
if __name__ == '__main__':
start = time.time()
src_path = "/media/tx-eva-data/NAS/基础数据库/华中科技大学同济医学院附属同济医院/2017_07_13_abnormal"
dec_path = "/media/tx-eva-data/NAS/原始数据库/华中科技大学同济医学院附属同济医院"
queue = multiprocessing.JoinableQueue()
list_p = []
for i in range(multiprocessing.cpu_count() - 2):
process = multiprocessing.Process(target=Worker, args=(queue, ))
process.daemon = True
process.start()
list_p.append(process)
for dirpath, dirnames, filenames in os.walk(src_path):
for file in filenames:
dcm_path = os.path.join(dirpath, file)
queue.put([dcm_path, dec_path])
queue.join()
for i in range(multiprocessing.cpuCount() - 2):
queue.put(None)
for p in list_p:
p.join()
elapsed = time.time() - start
print('main end! %f' % elapsed)
# USER-SPECIFIC PATHS AND VALUES
# We'll be using SWarp. Define the proper bin executable here. This is
# particularly important if you have different installations and want to use a
# specific version in the reduction. Also, note that the SWarp packaged with
# Scisoft probably won't work properly on Intel 64-bit Macs. It is strongly
# advised that you recompile and install SWarp from source if SWarp's coadd
# mosaics are null images.
swarp_bin = "/Applications/scisoft//i386/bin/swarp"
# We'll also be using some WCSTools. There should be no problem with the Scisoft
# version of thses.
sethead_bin = "/Applications/scisoft//i386/bin/sethead"
# For parallel processing we need the number of available CPUs. You could hard-
# code this value to something else, but it would result in non-optimal
# performance.
if doparallel == 1:
numprocessors = cpuCount()
else:
numprocessors = 1
# Debugging mode outputs intermediate image files to help track down problems.
# It requires more time to run because of the additional hard disk reads/writes.
DEBUG = False
#-------------------------------------------------------------------------------
# FUNCTION DEFINITIONS
# Simple function allowing parallelization of mosaicing with SWarp.
def run_swarp(command):
system(command)
return
# advised that you recompile and install SWarp from source if SWarp's coadd
# mosaics are null images.
swarp_bin = "swarp"
# We'll also be using some WCSTools. There should be no problem with the Scisoft
# version of thses.
sethead_bin = "sethead"
# For parallel processing we need the number of available CPUs. You could hard-
# code this value to something else, but it would result in non-optimal
# performance.
# AMORGAN ADDS for WCS FITTING Change if your python version is different.
# Will want to avoid having to load this in future versions.
pypath = ""
if doparallel == 1:
numprocessors = cpuCount()
else:
numprocessors = 1
# Debugging mode outputs intermediate image files to help track down problems.
# It requires more time to run because of the additional hard disk reads/writes.
DEBUG = False
#-------------------------------------------------------------------------------
# FUNCTION DEFINITIONS
# Simple function allowing parallelization of mosaicing with SWarp.
def run_swarp(command):
system(command)
return
#-------------------------------------------------------------------------------
# BEGIN MAIN PROGRAM
if not os.environ.has_key("Q_DIR"):
import version
__version__ = version.version
def complement(s, _comp=string.maketrans('ATCG', 'TAGC')):
return s.translate(_comp)
def revcomp(s, _comp=string.maketrans('ATCG', 'TAGC')):
return s.translate(_comp)[::-1]
CPU_COUNT = 4
try:
import multiprocessing as processing
CPU_COUNT = processing.cpu_count()
except ImportError:
import processing
CPU_COUNT = processing.cpuCount()
def write_c2t(fasta_name, unconverted, colorspace=False):
"""
given a fasta file, write a new file:
`some.fr.c2t.fasta` which contains:
+ the same headers prefixed with 'f' with all C's converted to T
+ headers prefixed with 'r' reverse complemented with
all C's converted to T.
if unconverted is false, then also save a file with the forward and reverse
without conversion.
"""
d = op.join(op.dirname(fasta_name), "bowtie_index")
if colorspace: d += "_colorspace"
if not op.exists(d): os.mkdir(d)
import test
# from test import pystone
import multiprocessing
import time
STONES_PER_PROCESS = 10 * test.pystone.LOOPS
def f(q):
t = test.pystone.pystones(STONES_PER_PROCESS)
q.put(t, block=True)
if __name__ == '__main__':
print 'multiprocessing test.pystones() benchmark'
print 'You have ' + str(multiprocessing.cpuCount()) + ' CPU(s)'
print 'Processes\tPystones\tWall time\tpystones/s'
results = multiprocessing.Queue()
for N in range(1, multiprocessing.cpuCount() + 3):
p = []
q = multiprocessing.Queue()
results = []
for m in range(1, N + 1):
p.append(multiprocessing.Process(target=f, args=(q, )))
start = time.time()
for pr in p:
pr.start()
for r in p:
def complement(s, _comp=string.maketrans('ATCG', 'TAGC')):
return s.translate(_comp)
def revcomp(s, _comp=string.maketrans('ATCG', 'TAGC')):
return s.translate(_comp)[::-1]
CPU_COUNT = 4
try:
import multiprocessing as processing
CPU_COUNT = processing.cpu_count()
except ImportError:
import processing
CPU_COUNT = processing.cpuCount()
def write_c2t(fasta_name, unconverted, colorspace=False):
"""
given a fasta file, write a new file:
`some.fr.c2t.fasta` which contains:
+ the same headers prefixed with 'f' with all C's converted to T
+ headers prefixed with 'r' reverse complemented with
all C's converted to T.
if unconverted is false, then also save a file with the forward and reverse
without conversion.
"""
d = op.join(op.dirname(fasta_name), "bowtie_index")
if colorspace: d += "_colorspace"