class SimpleTestCase(TestCase): def setUp(self): from redis import Redis from redis_semaphore import Semaphore self.client = Redis() self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem2 = Semaphore( client=self.client, count=self.s_count ) def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count
class SimpleTestCase(TestCase): def setUp(self): self.client = Redis() self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem1.reset() self.sem2 = Semaphore( client=self.client, count=self.s_count ) self.sem2.reset() def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0 def test_nonblocking(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem1.acquire() assert self.sem1.available_count == 0 self.sem1.blocking = False with self.assertRaises(NotAvailable): with self.sem1: assert False, 'should never reach here'
def seanomeGOstage2(workdir, jid, cutoffs): jobj = job.objects.get(pk=jid) sparams = json.loads(jobj.scriptparams) jobj.stage = STAGES_REVERSE['running'] jobj.save() setupUserEnv(workdir) close_old_connections() # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count=settings.REDIS_THREADS, namespace='SeanomeWorkers') if sparams.get("samples", "multi") == "multi": for l in open("sample_runner_script_2.sh"): if l.startswith("bash "): rdis_semaphore.acquire() l = l.strip().split() key = l[1].split("_")[0] v = cutoffs[key] v.sort() l = " ".join(l[:2]) if len(v) == 2: os.system("""%s -c %s -z %s >> log.out 2>> log.err """ % (l, v[0], v[1])) else: os.system("""%s %s %s >> log.out 2>> log.err """ % (l, 3, 200)) rdis_semaphore.release() rdis_semaphore.acquire() os.system( """bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """ % (str(sparams.get('minlen', settings.MIN_SEQ_LEN)), str(sparams.get('sim', settings.MIN_SEQ_SIM)))) if sparams.get("samples", "multi") == "multi": # TODO: will need to modify combineAln to deal with single case.. catdir = os.path.join(workdir, "concat_trimmed") for c in getSizes(workdir): os.system( """combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """ % dict(dbase=os.path.join(workdir, "csr", "seanome.db3"), cnt=c, outprefix=os.path.join(catdir, "%s_concat" % (c)))) # generate the FST histogram image os.system("""mkdir %s""" % (os.path.join(workdir, "images"))) os.system("""fst.py -d %s -w %s """ % (os.path.join( workdir, "csr", "seanome.db3"), os.path.join(workdir, "images"))) rdis_semaphore.release() jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() return True
def seanomeGOstage2(workdir, jid, cutoffs): jobj = job.objects.get(pk = jid) sparams = json.loads(jobj.scriptparams) jobj.stage = STAGES_REVERSE['running'] jobj.save() setupUserEnv(workdir) close_old_connections() # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers') if sparams.get("samples", "multi") == "multi": for l in open("sample_runner_script_2.sh"): if l.startswith("bash "): rdis_semaphore.acquire() l = l.strip().split() key = l[1].split("_")[0] v = cutoffs[key] v.sort() l = " ".join(l[:2]) if len(v) == 2: os.system("""%s -c %s -z %s >> log.out 2>> log.err """%(l, v[0], v[1])) else: os.system("""%s %s %s >> log.out 2>> log.err """%(l, 3, 200)) rdis_semaphore.release() rdis_semaphore.acquire() os.system("""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """%( str(sparams.get('minlen',settings.MIN_SEQ_LEN)) , str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) ) if sparams.get("samples", "multi") == "multi": # TODO: will need to modify combineAln to deal with single case.. catdir = os.path.join(workdir, "concat_trimmed") for c in getSizes(workdir): os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) ) # generate the FST histogram image os.system( """mkdir %s"""%( os.path.join(workdir, "images") ) ) os.system( """fst.py -d %s -w %s """ %(os.path.join(workdir, "csr", "seanome.db3"), os.path.join(workdir, "images") )) rdis_semaphore.release() jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() return True
def seanomeGO(workdir, jid, idmap): jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['running'] sparams = json.loads(jobj.scriptparams) jobj.save() os.system("""chmod -R 775 %s""" % (workdir)) setupUserEnv(workdir) close_old_connections() tasks = [] libs = {} coverageFiles = [] for k, row in idmap.iteritems(): infile_F = os.path.join(workdir, row[0]) infile_R = os.path.join(workdir, row[1]) try: infile_Ftmp = os.path.join(workdir, row[0].replace(" ", "_")) infile_Rtmp = os.path.join(workdir, row[1].replace(" ", "_")) os.rename(infile_F, infile_Ftmp) os.rename(infile_R, infile_Rtmp) infile_F = infile_Ftmp infile_R = infile_Rtmp libs[k] = [infile_F, infile_R] # We need this info for later.. coverageFiles.append( "%s" % (os.path.join(workdir, k, "%s.coverage.json" % (k)))) except: pass configfile = dict( libraries=libs, findcsr=dict(minlen=str(sparams.get('minlen', settings.MIN_SEQ_LEN)), msim=str(sparams.get('sim', settings.MIN_SEQ_SIM)))) with open(os.path.join(workdir, "config.yaml"), "w") as ymout: yaml.dump(configfile, ymout) with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout: if sparams.get("samples", "multi") != "multi" and len(libs) <= 2: coverageFiles = [ os.path.join(workdir, "csr", "combined.coverage.json") ] yaml.dump(dict(coverage=coverageFiles), ymout) # This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts if sparams.get("samples", "multi") == "multi": if (sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: if (sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count=settings.REDIS_THREADS, namespace='SeanomeWorkers') rdis_semaphore.acquire() catdir = os.path.join(workdir, "concat_trimmed") if not jobj.interrupt: os.system("""bash primary_script.sh >> log.out 2>> log.err""") for c in getSizes(workdir): os.system( """combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """ % dict(dbase=os.path.join(workdir, "csr", "seanome.db3"), cnt=c, outprefix=os.path.join(catdir, "%s_concat" % (c)))) jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() else: os.system( """bash sample_runner_script_1.sh >> log.out 2>> log.err """) if sparams.get("samples", "multi") != "multi": os.system( """bash sample_runner_script_2.sh >> log.out 2>> log.err """) jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s1'] jobj.save() rdis_semaphore.release() return True
def seanomeGO(workdir, jid, idmap): jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['running'] sparams = json.loads(jobj.scriptparams) jobj.save() os.system("""chmod -R 775 %s"""%(workdir)) setupUserEnv(workdir) close_old_connections() tasks = [] libs = {} coverageFiles =[] for k, row in idmap.iteritems(): infile_F = os.path.join(workdir, row[0]) infile_R = os.path.join(workdir, row[1]) try: infile_Ftmp = os.path.join(workdir, row[0].replace(" ","_")) infile_Rtmp = os.path.join(workdir, row[1].replace(" ","_")) os.rename(infile_F, infile_Ftmp) os.rename(infile_R, infile_Rtmp) infile_F = infile_Ftmp infile_R = infile_Rtmp libs[k] = [infile_F, infile_R] # We need this info for later.. coverageFiles.append("%s"%(os.path.join(workdir, k, "%s.coverage.json"%(k)) ) ) except: pass configfile = dict(libraries=libs, findcsr = dict( minlen = str(sparams.get('minlen',settings.MIN_SEQ_LEN) ) , msim = str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) ) with open(os.path.join(workdir, "config.yaml"), "w") as ymout: yaml.dump(configfile, ymout) with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout: if sparams.get("samples", "multi") != "multi" and len(libs) <= 2: coverageFiles = [os.path.join(workdir, "csr", "combined.coverage.json") ] yaml.dump(dict(coverage=coverageFiles), ymout) # This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts if sparams.get("samples", "multi") == "multi": if(sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: if(sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers') rdis_semaphore.acquire() catdir = os.path.join(workdir, "concat_trimmed") if not jobj.interrupt: os.system("""bash primary_script.sh >> log.out 2>> log.err""") for c in getSizes(workdir): os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) ) jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() else: os.system("""bash sample_runner_script_1.sh >> log.out 2>> log.err """) if sparams.get("samples", "multi") != "multi": os.system("""bash sample_runner_script_2.sh >> log.out 2>> log.err """) jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s1'] jobj.save() rdis_semaphore.release() return True
class SimpleTestCase(TestCase): def setUp(self): redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost' self.client = StrictRedis(host=redis_host) self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem1.reset() self.sem2 = Semaphore( client=self.client, count=self.s_count ) self.sem2.reset() self.sem3 = Semaphore( client=self.client, count=self.s_count, blocking=False ) self.sem3.reset() def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0 def test_nonblocking(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem3.acquire() assert self.sem3.available_count == 0 with self.assertRaises(NotAvailable): with self.sem3: assert False, 'should never reach here' def test_acquire_with_timeout(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem1.acquire() with self.assertRaises(NotAvailable): self.sem1.acquire(timeout=1)