class SimpleTestCase(TestCase):
def setUp(self):
from redis import Redis
from redis_semaphore import Semaphore
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
class SimpleTestCase(TestCase):
def setUp(self):
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem1.reset()
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2.reset()
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def test_nonblocking(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem1.acquire()
assert self.sem1.available_count == 0
self.sem1.blocking = False
with self.assertRaises(NotAvailable):
with self.sem1:
assert False, 'should never reach here'
def seanomeGOstage2(workdir, jid, cutoffs):
jobj = job.objects.get(pk=jid)
sparams = json.loads(jobj.scriptparams)
jobj.stage = STAGES_REVERSE['running']
jobj.save()
setupUserEnv(workdir)
close_old_connections()
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(),
count=settings.REDIS_THREADS,
namespace='SeanomeWorkers')
if sparams.get("samples", "multi") == "multi":
for l in open("sample_runner_script_2.sh"):
if l.startswith("bash "):
rdis_semaphore.acquire()
l = l.strip().split()
key = l[1].split("_")[0]
v = cutoffs[key]
v.sort()
l = " ".join(l[:2])
if len(v) == 2:
os.system("""%s -c %s -z %s >> log.out 2>> log.err """ %
(l, v[0], v[1]))
else:
os.system("""%s %s %s >> log.out 2>> log.err """ %
(l, 3, 200))
rdis_semaphore.release()
rdis_semaphore.acquire()
os.system(
"""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """
% (str(sparams.get('minlen', settings.MIN_SEQ_LEN)),
str(sparams.get('sim', settings.MIN_SEQ_SIM))))
if sparams.get("samples", "multi") == "multi":
# TODO: will need to modify combineAln to deal with single case..
catdir = os.path.join(workdir, "concat_trimmed")
for c in getSizes(workdir):
os.system(
"""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """
% dict(dbase=os.path.join(workdir, "csr", "seanome.db3"),
cnt=c,
outprefix=os.path.join(catdir, "%s_concat" % (c))))
# generate the FST histogram image
os.system("""mkdir %s""" % (os.path.join(workdir, "images")))
os.system("""fst.py -d %s -w %s """ % (os.path.join(
workdir, "csr", "seanome.db3"), os.path.join(workdir, "images")))
rdis_semaphore.release()
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
return True
def seanomeGOstage2(workdir, jid, cutoffs):
jobj = job.objects.get(pk = jid)
sparams = json.loads(jobj.scriptparams)
jobj.stage = STAGES_REVERSE['running']
jobj.save()
setupUserEnv(workdir)
close_old_connections()
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers')
if sparams.get("samples", "multi") == "multi":
for l in open("sample_runner_script_2.sh"):
if l.startswith("bash "):
rdis_semaphore.acquire()
l = l.strip().split()
key = l[1].split("_")[0]
v = cutoffs[key]
v.sort()
l = " ".join(l[:2])
if len(v) == 2:
os.system("""%s -c %s -z %s >> log.out 2>> log.err """%(l, v[0], v[1]))
else:
os.system("""%s %s %s >> log.out 2>> log.err """%(l, 3, 200))
rdis_semaphore.release()
rdis_semaphore.acquire()
os.system("""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """%( str(sparams.get('minlen',settings.MIN_SEQ_LEN)) , str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) )
if sparams.get("samples", "multi") == "multi":
# TODO: will need to modify combineAln to deal with single case..
catdir = os.path.join(workdir, "concat_trimmed")
for c in getSizes(workdir):
os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) )
# generate the FST histogram image
os.system( """mkdir %s"""%( os.path.join(workdir, "images") ) )
os.system( """fst.py -d %s -w %s """ %(os.path.join(workdir, "csr", "seanome.db3"), os.path.join(workdir, "images") ))
rdis_semaphore.release()
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
return True
def seanomeGO(workdir, jid, idmap):
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['running']
sparams = json.loads(jobj.scriptparams)
jobj.save()
os.system("""chmod -R 775 %s""" % (workdir))
setupUserEnv(workdir)
close_old_connections()
tasks = []
libs = {}
coverageFiles = []
for k, row in idmap.iteritems():
infile_F = os.path.join(workdir, row[0])
infile_R = os.path.join(workdir, row[1])
try:
infile_Ftmp = os.path.join(workdir, row[0].replace(" ", "_"))
infile_Rtmp = os.path.join(workdir, row[1].replace(" ", "_"))
os.rename(infile_F, infile_Ftmp)
os.rename(infile_R, infile_Rtmp)
infile_F = infile_Ftmp
infile_R = infile_Rtmp
libs[k] = [infile_F, infile_R]
# We need this info for later..
coverageFiles.append(
"%s" % (os.path.join(workdir, k, "%s.coverage.json" % (k))))
except:
pass
configfile = dict(
libraries=libs,
findcsr=dict(minlen=str(sparams.get('minlen', settings.MIN_SEQ_LEN)),
msim=str(sparams.get('sim', settings.MIN_SEQ_SIM))))
with open(os.path.join(workdir, "config.yaml"), "w") as ymout:
yaml.dump(configfile, ymout)
with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout:
if sparams.get("samples", "multi") != "multi" and len(libs) <= 2:
coverageFiles = [
os.path.join(workdir, "csr", "combined.coverage.json")
]
yaml.dump(dict(coverage=coverageFiles), ymout)
# This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts
if sparams.get("samples", "multi") == "multi":
if (sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
if (sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(),
count=settings.REDIS_THREADS,
namespace='SeanomeWorkers')
rdis_semaphore.acquire()
catdir = os.path.join(workdir, "concat_trimmed")
if not jobj.interrupt:
os.system("""bash primary_script.sh >> log.out 2>> log.err""")
for c in getSizes(workdir):
os.system(
"""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """
% dict(dbase=os.path.join(workdir, "csr", "seanome.db3"),
cnt=c,
outprefix=os.path.join(catdir, "%s_concat" % (c))))
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
else:
os.system(
"""bash sample_runner_script_1.sh >> log.out 2>> log.err """)
if sparams.get("samples", "multi") != "multi":
os.system(
"""bash sample_runner_script_2.sh >> log.out 2>> log.err """)
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s1']
jobj.save()
rdis_semaphore.release()
return True
def seanomeGO(workdir, jid, idmap):
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['running']
sparams = json.loads(jobj.scriptparams)
jobj.save()
os.system("""chmod -R 775 %s"""%(workdir))
setupUserEnv(workdir)
close_old_connections()
tasks = []
libs = {}
coverageFiles =[]
for k, row in idmap.iteritems():
infile_F = os.path.join(workdir, row[0])
infile_R = os.path.join(workdir, row[1])
try:
infile_Ftmp = os.path.join(workdir, row[0].replace(" ","_"))
infile_Rtmp = os.path.join(workdir, row[1].replace(" ","_"))
os.rename(infile_F, infile_Ftmp)
os.rename(infile_R, infile_Rtmp)
infile_F = infile_Ftmp
infile_R = infile_Rtmp
libs[k] = [infile_F, infile_R]
# We need this info for later..
coverageFiles.append("%s"%(os.path.join(workdir, k, "%s.coverage.json"%(k)) ) )
except:
pass
configfile = dict(libraries=libs, findcsr = dict( minlen = str(sparams.get('minlen',settings.MIN_SEQ_LEN) ) , msim = str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) )
with open(os.path.join(workdir, "config.yaml"), "w") as ymout:
yaml.dump(configfile, ymout)
with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout:
if sparams.get("samples", "multi") != "multi" and len(libs) <= 2:
coverageFiles = [os.path.join(workdir, "csr", "combined.coverage.json") ]
yaml.dump(dict(coverage=coverageFiles), ymout)
# This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts
if sparams.get("samples", "multi") == "multi":
if(sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
if(sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers')
rdis_semaphore.acquire()
catdir = os.path.join(workdir, "concat_trimmed")
if not jobj.interrupt:
os.system("""bash primary_script.sh >> log.out 2>> log.err""")
for c in getSizes(workdir):
os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) )
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
else:
os.system("""bash sample_runner_script_1.sh >> log.out 2>> log.err """)
if sparams.get("samples", "multi") != "multi":
os.system("""bash sample_runner_script_2.sh >> log.out 2>> log.err """)
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s1']
jobj.save()
rdis_semaphore.release()
return True
class SimpleTestCase(TestCase):
def setUp(self):
redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost'
self.client = StrictRedis(host=redis_host)
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem1.reset()
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2.reset()
self.sem3 = Semaphore(
client=self.client,
count=self.s_count,
blocking=False
)
self.sem3.reset()
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def test_nonblocking(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem3.acquire()
assert self.sem3.available_count == 0
with self.assertRaises(NotAvailable):
with self.sem3:
assert False, 'should never reach here'
def test_acquire_with_timeout(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem1.acquire()
with self.assertRaises(NotAvailable):
self.sem1.acquire(timeout=1)
