def seanomeGOstage2(workdir, jid, cutoffs): jobj = job.objects.get(pk=jid) sparams = json.loads(jobj.scriptparams) jobj.stage = STAGES_REVERSE['running'] jobj.save() setupUserEnv(workdir) close_old_connections() # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count=settings.REDIS_THREADS, namespace='SeanomeWorkers') if sparams.get("samples", "multi") == "multi": for l in open("sample_runner_script_2.sh"): if l.startswith("bash "): rdis_semaphore.acquire() l = l.strip().split() key = l[1].split("_")[0] v = cutoffs[key] v.sort() l = " ".join(l[:2]) if len(v) == 2: os.system("""%s -c %s -z %s >> log.out 2>> log.err """ % (l, v[0], v[1])) else: os.system("""%s %s %s >> log.out 2>> log.err """ % (l, 3, 200)) rdis_semaphore.release() rdis_semaphore.acquire() os.system( """bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """ % (str(sparams.get('minlen', settings.MIN_SEQ_LEN)), str(sparams.get('sim', settings.MIN_SEQ_SIM)))) if sparams.get("samples", "multi") == "multi": # TODO: will need to modify combineAln to deal with single case.. catdir = os.path.join(workdir, "concat_trimmed") for c in getSizes(workdir): os.system( """combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """ % dict(dbase=os.path.join(workdir, "csr", "seanome.db3"), cnt=c, outprefix=os.path.join(catdir, "%s_concat" % (c)))) # generate the FST histogram image os.system("""mkdir %s""" % (os.path.join(workdir, "images"))) os.system("""fst.py -d %s -w %s """ % (os.path.join( workdir, "csr", "seanome.db3"), os.path.join(workdir, "images"))) rdis_semaphore.release() jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() return True
def setUp(self): from redis import Redis from redis_semaphore import Semaphore self.client = Redis() self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem2 = Semaphore( client=self.client, count=self.s_count )
def setUp(self): self.client = Redis() self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem1.reset() self.sem2 = Semaphore( client=self.client, count=self.s_count ) self.sem2.reset()
def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0
def get_semaphor(namespace, count=1): from redis import Redis from redis_semaphore import Semaphore from urllib.parse import urlparse redis = urlparse(settings.SEMAPHORE_REDIS_URL) semaphore = Semaphore(Redis(host=redis.hostname, port=redis.port), count=count, namespace=namespace) return semaphore
def get_concurrent_redshift_query_queue_semaphore(queue_name): concurrency = settings.REDSHIFT_QUERY_QUEUES[queue_name]["concurrency"] concurrent_redshift_query_semaphore = Semaphore( redshift.get_redshift_cache_redis_client(), count=concurrency, namespace=queue_name, stale_client_timeout=300, blocking=False) return concurrent_redshift_query_semaphore
class SimpleTestCase(TestCase): def setUp(self): from redis import Redis from redis_semaphore import Semaphore self.client = Redis() self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem2 = Semaphore( client=self.client, count=self.s_count ) def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count
def seanomeGOstage2(workdir, jid, cutoffs): jobj = job.objects.get(pk = jid) sparams = json.loads(jobj.scriptparams) jobj.stage = STAGES_REVERSE['running'] jobj.save() setupUserEnv(workdir) close_old_connections() # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers') if sparams.get("samples", "multi") == "multi": for l in open("sample_runner_script_2.sh"): if l.startswith("bash "): rdis_semaphore.acquire() l = l.strip().split() key = l[1].split("_")[0] v = cutoffs[key] v.sort() l = " ".join(l[:2]) if len(v) == 2: os.system("""%s -c %s -z %s >> log.out 2>> log.err """%(l, v[0], v[1])) else: os.system("""%s %s %s >> log.out 2>> log.err """%(l, 3, 200)) rdis_semaphore.release() rdis_semaphore.acquire() os.system("""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """%( str(sparams.get('minlen',settings.MIN_SEQ_LEN)) , str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) ) if sparams.get("samples", "multi") == "multi": # TODO: will need to modify combineAln to deal with single case.. catdir = os.path.join(workdir, "concat_trimmed") for c in getSizes(workdir): os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) ) # generate the FST histogram image os.system( """mkdir %s"""%( os.path.join(workdir, "images") ) ) os.system( """fst.py -d %s -w %s """ %(os.path.join(workdir, "csr", "seanome.db3"), os.path.join(workdir, "images") )) rdis_semaphore.release() jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() return True
class SimpleTestCase(TestCase): def setUp(self): self.client = Redis() self.s_count = 2 self.sem1 = Semaphore(client=self.client, count=self.s_count) self.sem2 = Semaphore(client=self.client, count=self.s_count) def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0 def test_nonblocking(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem1.acquire() assert self.sem1.available_count == 0 self.sem1.blocking = False with self.assertRaises(NotAvailable): with self.sem1: assert False, "should never reach here"
class SimpleTestCase(TestCase): def setUp(self): self.client = Redis() self.s_count = 2 self.sem1 = Semaphore(client=self.client, count=self.s_count) self.sem2 = Semaphore(client=self.client, count=self.s_count) def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0 def test_nonblocking(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem1.acquire() assert self.sem1.available_count == 0 self.sem1.blocking = False with self.assertRaises(NotAvailable): with self.sem1: assert False, 'should never reach here'
def get_semaphore(namespace, count=1, db=None, blocking=False, stale_client_timeout=60): from redis import Redis from redis_semaphore import Semaphore from urllib.parse import urlparse redis = urlparse(settings.SEMAPHORE_REDIS_URL) if db is None: db = int(redis.path.lstrip('/')) semaphore = Semaphore( Redis(host=redis.hostname, port=redis.port, db=db), count=count, namespace=namespace, blocking=blocking, stale_client_timeout=stale_client_timeout, ) return semaphore
def setUp(self): redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost' self.client = StrictRedis(host=redis_host) self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem1.reset() self.sem2 = Semaphore( client=self.client, count=self.s_count ) self.sem2.reset() self.sem3 = Semaphore( client=self.client, count=self.s_count, blocking=False ) self.sem3.reset()
#!/usr/bin/env python # Copyright (C) 2016 Deloitte Argentina. # This file is part of CodexGigas - https://github.com/codexgigassys/ # See the file 'LICENSE' for copying permission. # # In the unfortunate case that a worker is forcefully # stopped while having the semaphore, all workers # will lock. The chance of this happening is low # but can happen (and has happened). # Here is the code to release the semaphore. # import pathmagic from redis import Redis from redis_semaphore import Semaphore from threading import Thread from db_pool import * semaphore = Semaphore(Redis(host=envget('redis.host')), count=1, namespace='example') token = semaphore.get_namespaced_key('example') semaphore.signal(token)
def __init__(self): self.semaphore = Semaphore(Redis(host=envget('redis.host')), count=1, namespace='example')
#-*- coding:utf-8 -*- from redis import Redis from redis_semaphore import Semaphore from threading import Thread import urllib2 import time semaphore = Semaphore(Redis(), count=2, namespace='example') def task(i): url = 'https://www.google.co.jp/' with semaphore: print('id: {} => {}'.format(i, urllib2.urlopen(url).code)) print('sleep...') time.sleep(2) def main(): threads = list() for i in range(5): threads.append(Thread(target=task, args=(i,))) for th in threads: th.start() for th in threads: th.join() if __name__ == '__main__': main()
parser.add_argument( '-mq', '--rabbitmq', dest='host_mq', help='host rabbit [localhost]', default="localhost", ) parser.add_argument( '-r', '--routing', dest='routing', help='routing key [client1]', default="client1", ) args = parser.parse_args() semaphore = Semaphore(Redis(), count=1, namespace='sem1') semaphore_con = Semaphore(Redis(), count=1, namespace='sem_con') app = Flask(__name__) app.threaded = True io = SocketIO(app) pool = redis.ConnectionPool(host='localhost', port=6379, db=0) eventlet.monkey_patch() consumers = {} @io.on('return_response') def return_response(message): r = redis.Redis(connection_pool=pool) r.rpush(message["uuid"], json.dumps(message["result"]))
class SimpleTestCase(TestCase): def setUp(self): redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost' self.client = StrictRedis(host=redis_host) self.s_count = 2 self.sem1 = Semaphore( client=self.client, count=self.s_count ) self.sem1.reset() self.sem2 = Semaphore( client=self.client, count=self.s_count ) self.sem2.reset() self.sem3 = Semaphore( client=self.client, count=self.s_count, blocking=False ) self.sem3.reset() def test_lock(self): assert self.sem1.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem1.available_count == (self.s_count - 1) self.sem1.release() assert self.sem1.available_count == self.s_count assert self.sem2.available_count == self.s_count assert self.sem1.acquire() is not None assert self.sem2.available_count == (self.s_count - 1) self.sem1.release() def test_with(self): assert self.sem1.available_count == self.s_count with self.sem1 as sem: assert sem.available_count == (self.s_count - 1) with sem: assert sem.available_count == (self.s_count - 2) assert self.sem1.available_count == self.s_count def test_create_with_existing(self): with self.sem1 as sem1: with self.sem2 as sem2: assert sem1.available_count == 0 assert sem2.available_count == 0 sem3 = Semaphore(client=self.client, count=40) assert sem3.available_count == 0 def test_nonblocking(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem3.acquire() assert self.sem3.available_count == 0 with self.assertRaises(NotAvailable): with self.sem3: assert False, 'should never reach here' def test_acquire_with_timeout(self): from redis_semaphore import NotAvailable for _ in range(self.s_count): self.sem1.acquire() with self.assertRaises(NotAvailable): self.sem1.acquire(timeout=1)
def seanomeGO(workdir, jid, idmap): jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['running'] sparams = json.loads(jobj.scriptparams) jobj.save() os.system("""chmod -R 775 %s"""%(workdir)) setupUserEnv(workdir) close_old_connections() tasks = [] libs = {} coverageFiles =[] for k, row in idmap.iteritems(): infile_F = os.path.join(workdir, row[0]) infile_R = os.path.join(workdir, row[1]) try: infile_Ftmp = os.path.join(workdir, row[0].replace(" ","_")) infile_Rtmp = os.path.join(workdir, row[1].replace(" ","_")) os.rename(infile_F, infile_Ftmp) os.rename(infile_R, infile_Rtmp) infile_F = infile_Ftmp infile_R = infile_Rtmp libs[k] = [infile_F, infile_R] # We need this info for later.. coverageFiles.append("%s"%(os.path.join(workdir, k, "%s.coverage.json"%(k)) ) ) except: pass configfile = dict(libraries=libs, findcsr = dict( minlen = str(sparams.get('minlen',settings.MIN_SEQ_LEN) ) , msim = str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) ) with open(os.path.join(workdir, "config.yaml"), "w") as ymout: yaml.dump(configfile, ymout) with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout: if sparams.get("samples", "multi") != "multi" and len(libs) <= 2: coverageFiles = [os.path.join(workdir, "csr", "combined.coverage.json") ] yaml.dump(dict(coverage=coverageFiles), ymout) # This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts if sparams.get("samples", "multi") == "multi": if(sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: if(sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS)) # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers') rdis_semaphore.acquire() catdir = os.path.join(workdir, "concat_trimmed") if not jobj.interrupt: os.system("""bash primary_script.sh >> log.out 2>> log.err""") for c in getSizes(workdir): os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) ) jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() else: os.system("""bash sample_runner_script_1.sh >> log.out 2>> log.err """) if sparams.get("samples", "multi") != "multi": os.system("""bash sample_runner_script_2.sh >> log.out 2>> log.err """) jobj = job.objects.get(pk = jid) jobj.stage = STAGES_REVERSE['done-s1'] jobj.save() rdis_semaphore.release() return True
def seanomeGO(workdir, jid, idmap): jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['running'] sparams = json.loads(jobj.scriptparams) jobj.save() os.system("""chmod -R 775 %s""" % (workdir)) setupUserEnv(workdir) close_old_connections() tasks = [] libs = {} coverageFiles = [] for k, row in idmap.iteritems(): infile_F = os.path.join(workdir, row[0]) infile_R = os.path.join(workdir, row[1]) try: infile_Ftmp = os.path.join(workdir, row[0].replace(" ", "_")) infile_Rtmp = os.path.join(workdir, row[1].replace(" ", "_")) os.rename(infile_F, infile_Ftmp) os.rename(infile_R, infile_Rtmp) infile_F = infile_Ftmp infile_R = infile_Rtmp libs[k] = [infile_F, infile_R] # We need this info for later.. coverageFiles.append( "%s" % (os.path.join(workdir, k, "%s.coverage.json" % (k)))) except: pass configfile = dict( libraries=libs, findcsr=dict(minlen=str(sparams.get('minlen', settings.MIN_SEQ_LEN)), msim=str(sparams.get('sim', settings.MIN_SEQ_SIM)))) with open(os.path.join(workdir, "config.yaml"), "w") as ymout: yaml.dump(configfile, ymout) with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout: if sparams.get("samples", "multi") != "multi" and len(libs) <= 2: coverageFiles = [ os.path.join(workdir, "csr", "combined.coverage.json") ] yaml.dump(dict(coverage=coverageFiles), ymout) # This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts if sparams.get("samples", "multi") == "multi": if (sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: if (sparams.get('repeatmask', 0) == 1): os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) else: os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single" % (os.path.join(workdir, "config.yaml"), "seanome.db3", settings.WORK_THREADS)) # used in an attempt to limit how many threads of processing a task spins up at any given time. rdis_semaphore = Semaphore(Redis(), count=settings.REDIS_THREADS, namespace='SeanomeWorkers') rdis_semaphore.acquire() catdir = os.path.join(workdir, "concat_trimmed") if not jobj.interrupt: os.system("""bash primary_script.sh >> log.out 2>> log.err""") for c in getSizes(workdir): os.system( """combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """ % dict(dbase=os.path.join(workdir, "csr", "seanome.db3"), cnt=c, outprefix=os.path.join(catdir, "%s_concat" % (c)))) jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s2'] jobj.save() else: os.system( """bash sample_runner_script_1.sh >> log.out 2>> log.err """) if sparams.get("samples", "multi") != "multi": os.system( """bash sample_runner_script_2.sh >> log.out 2>> log.err """) jobj = job.objects.get(pk=jid) jobj.stage = STAGES_REVERSE['done-s1'] jobj.save() rdis_semaphore.release() return True