def seanomeGOstage2(workdir, jid, cutoffs):
jobj = job.objects.get(pk=jid)
sparams = json.loads(jobj.scriptparams)
jobj.stage = STAGES_REVERSE['running']
jobj.save()
setupUserEnv(workdir)
close_old_connections()
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(),
count=settings.REDIS_THREADS,
namespace='SeanomeWorkers')
if sparams.get("samples", "multi") == "multi":
for l in open("sample_runner_script_2.sh"):
if l.startswith("bash "):
rdis_semaphore.acquire()
l = l.strip().split()
key = l[1].split("_")[0]
v = cutoffs[key]
v.sort()
l = " ".join(l[:2])
if len(v) == 2:
os.system("""%s -c %s -z %s >> log.out 2>> log.err """ %
(l, v[0], v[1]))
else:
os.system("""%s %s %s >> log.out 2>> log.err """ %
(l, 3, 200))
rdis_semaphore.release()
rdis_semaphore.acquire()
os.system(
"""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """
% (str(sparams.get('minlen', settings.MIN_SEQ_LEN)),
str(sparams.get('sim', settings.MIN_SEQ_SIM))))
if sparams.get("samples", "multi") == "multi":
# TODO: will need to modify combineAln to deal with single case..
catdir = os.path.join(workdir, "concat_trimmed")
for c in getSizes(workdir):
os.system(
"""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """
% dict(dbase=os.path.join(workdir, "csr", "seanome.db3"),
cnt=c,
outprefix=os.path.join(catdir, "%s_concat" % (c))))
# generate the FST histogram image
os.system("""mkdir %s""" % (os.path.join(workdir, "images")))
os.system("""fst.py -d %s -w %s """ % (os.path.join(
workdir, "csr", "seanome.db3"), os.path.join(workdir, "images")))
rdis_semaphore.release()
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
return True
def setUp(self):
from redis import Redis
from redis_semaphore import Semaphore
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
def setUp(self):
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem1.reset()
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2.reset()
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def get_semaphor(namespace, count=1):
from redis import Redis
from redis_semaphore import Semaphore
from urllib.parse import urlparse
redis = urlparse(settings.SEMAPHORE_REDIS_URL)
semaphore = Semaphore(Redis(host=redis.hostname, port=redis.port), count=count, namespace=namespace)
return semaphore
def get_concurrent_redshift_query_queue_semaphore(queue_name):
concurrency = settings.REDSHIFT_QUERY_QUEUES[queue_name]["concurrency"]
concurrent_redshift_query_semaphore = Semaphore(
redshift.get_redshift_cache_redis_client(),
count=concurrency,
namespace=queue_name,
stale_client_timeout=300,
blocking=False)
return concurrent_redshift_query_semaphore
class SimpleTestCase(TestCase):
def setUp(self):
from redis import Redis
from redis_semaphore import Semaphore
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def seanomeGOstage2(workdir, jid, cutoffs):
jobj = job.objects.get(pk = jid)
sparams = json.loads(jobj.scriptparams)
jobj.stage = STAGES_REVERSE['running']
jobj.save()
setupUserEnv(workdir)
close_old_connections()
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers')
if sparams.get("samples", "multi") == "multi":
for l in open("sample_runner_script_2.sh"):
if l.startswith("bash "):
rdis_semaphore.acquire()
l = l.strip().split()
key = l[1].split("_")[0]
v = cutoffs[key]
v.sort()
l = " ".join(l[:2])
if len(v) == 2:
os.system("""%s -c %s -z %s >> log.out 2>> log.err """%(l, v[0], v[1]))
else:
os.system("""%s %s %s >> log.out 2>> log.err """%(l, 3, 200))
rdis_semaphore.release()
rdis_semaphore.acquire()
os.system("""bash sample_runner_script_3.sh -l %s -s %s >> log.out 2>> log.err """%( str(sparams.get('minlen',settings.MIN_SEQ_LEN)) , str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) )
if sparams.get("samples", "multi") == "multi":
# TODO: will need to modify combineAln to deal with single case..
catdir = os.path.join(workdir, "concat_trimmed")
for c in getSizes(workdir):
os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) )
# generate the FST histogram image
os.system( """mkdir %s"""%( os.path.join(workdir, "images") ) )
os.system( """fst.py -d %s -w %s """ %(os.path.join(workdir, "csr", "seanome.db3"), os.path.join(workdir, "images") ))
rdis_semaphore.release()
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
return True
class SimpleTestCase(TestCase):
def setUp(self):
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(client=self.client, count=self.s_count)
self.sem2 = Semaphore(client=self.client, count=self.s_count)
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def test_nonblocking(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem1.acquire()
assert self.sem1.available_count == 0
self.sem1.blocking = False
with self.assertRaises(NotAvailable):
with self.sem1:
assert False, "should never reach here"
class SimpleTestCase(TestCase):
def setUp(self):
self.client = Redis()
self.s_count = 2
self.sem1 = Semaphore(client=self.client, count=self.s_count)
self.sem2 = Semaphore(client=self.client, count=self.s_count)
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def test_nonblocking(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem1.acquire()
assert self.sem1.available_count == 0
self.sem1.blocking = False
with self.assertRaises(NotAvailable):
with self.sem1:
assert False, 'should never reach here'
def get_semaphore(namespace, count=1, db=None, blocking=False, stale_client_timeout=60):
from redis import Redis
from redis_semaphore import Semaphore
from urllib.parse import urlparse
redis = urlparse(settings.SEMAPHORE_REDIS_URL)
if db is None:
db = int(redis.path.lstrip('/'))
semaphore = Semaphore(
Redis(host=redis.hostname, port=redis.port, db=db),
count=count,
namespace=namespace,
blocking=blocking,
stale_client_timeout=stale_client_timeout,
)
return semaphore
def setUp(self):
redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost'
self.client = StrictRedis(host=redis_host)
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem1.reset()
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2.reset()
self.sem3 = Semaphore(
client=self.client,
count=self.s_count,
blocking=False
)
self.sem3.reset()
#!/usr/bin/env python
# Copyright (C) 2016 Deloitte Argentina.
# This file is part of CodexGigas - https://github.com/codexgigassys/
# See the file 'LICENSE' for copying permission.
#
# In the unfortunate case that a worker is forcefully
# stopped while having the semaphore, all workers
# will lock. The chance of this happening is low
# but can happen (and has happened).
# Here is the code to release the semaphore.
#
import pathmagic
from redis import Redis
from redis_semaphore import Semaphore
from threading import Thread
from db_pool import *
semaphore = Semaphore(Redis(host=envget('redis.host')),
count=1,
namespace='example')
token = semaphore.get_namespaced_key('example')
semaphore.signal(token)
def __init__(self):
self.semaphore = Semaphore(Redis(host=envget('redis.host')),
count=1,
namespace='example')
#!/usr/bin/env python
# Copyright (C) 2016 Deloitte Argentina.
# This file is part of CodexGigas - https://github.com/codexgigassys/
# See the file 'LICENSE' for copying permission.
#
# In the unfortunate case that a worker is forcefully
# stopped while having the semaphore, all workers
# will lock. The chance of this happening is low
# but can happen (and has happened).
# Here is the code to release the semaphore.
#
import pathmagic
from redis import Redis
from redis_semaphore import Semaphore
from threading import Thread
from db_pool import *
semaphore = Semaphore(Redis(host=envget('redis.host')),
count=1, namespace='example')
token = semaphore.get_namespaced_key('example')
semaphore.signal(token)
#-*- coding:utf-8 -*-
from redis import Redis
from redis_semaphore import Semaphore
from threading import Thread
import urllib2
import time
semaphore = Semaphore(Redis(), count=2, namespace='example')
def task(i):
url = 'https://www.google.co.jp/'
with semaphore:
print('id: {} => {}'.format(i, urllib2.urlopen(url).code))
print('sleep...')
time.sleep(2)
def main():
threads = list()
for i in range(5):
threads.append(Thread(target=task, args=(i,)))
for th in threads:
th.start()
for th in threads:
th.join()
if __name__ == '__main__':
main()
parser.add_argument(
'-mq',
'--rabbitmq',
dest='host_mq',
help='host rabbit [localhost]',
default="localhost",
)
parser.add_argument(
'-r',
'--routing',
dest='routing',
help='routing key [client1]',
default="client1",
)
args = parser.parse_args()
semaphore = Semaphore(Redis(), count=1, namespace='sem1')
semaphore_con = Semaphore(Redis(), count=1, namespace='sem_con')
app = Flask(__name__)
app.threaded = True
io = SocketIO(app)
pool = redis.ConnectionPool(host='localhost', port=6379, db=0)
eventlet.monkey_patch()
consumers = {}
@io.on('return_response')
def return_response(message):
r = redis.Redis(connection_pool=pool)
r.rpush(message["uuid"], json.dumps(message["result"]))
class SimpleTestCase(TestCase):
def setUp(self):
redis_host = os.environ.get('TEST_REDIS_HOST') or 'localhost'
self.client = StrictRedis(host=redis_host)
self.s_count = 2
self.sem1 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem1.reset()
self.sem2 = Semaphore(
client=self.client,
count=self.s_count
)
self.sem2.reset()
self.sem3 = Semaphore(
client=self.client,
count=self.s_count,
blocking=False
)
self.sem3.reset()
def test_lock(self):
assert self.sem1.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem1.available_count == (self.s_count - 1)
self.sem1.release()
assert self.sem1.available_count == self.s_count
assert self.sem2.available_count == self.s_count
assert self.sem1.acquire() is not None
assert self.sem2.available_count == (self.s_count - 1)
self.sem1.release()
def test_with(self):
assert self.sem1.available_count == self.s_count
with self.sem1 as sem:
assert sem.available_count == (self.s_count - 1)
with sem:
assert sem.available_count == (self.s_count - 2)
assert self.sem1.available_count == self.s_count
def test_create_with_existing(self):
with self.sem1 as sem1:
with self.sem2 as sem2:
assert sem1.available_count == 0
assert sem2.available_count == 0
sem3 = Semaphore(client=self.client, count=40)
assert sem3.available_count == 0
def test_nonblocking(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem3.acquire()
assert self.sem3.available_count == 0
with self.assertRaises(NotAvailable):
with self.sem3:
assert False, 'should never reach here'
def test_acquire_with_timeout(self):
from redis_semaphore import NotAvailable
for _ in range(self.s_count):
self.sem1.acquire()
with self.assertRaises(NotAvailable):
self.sem1.acquire(timeout=1)
def seanomeGO(workdir, jid, idmap):
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['running']
sparams = json.loads(jobj.scriptparams)
jobj.save()
os.system("""chmod -R 775 %s"""%(workdir))
setupUserEnv(workdir)
close_old_connections()
tasks = []
libs = {}
coverageFiles =[]
for k, row in idmap.iteritems():
infile_F = os.path.join(workdir, row[0])
infile_R = os.path.join(workdir, row[1])
try:
infile_Ftmp = os.path.join(workdir, row[0].replace(" ","_"))
infile_Rtmp = os.path.join(workdir, row[1].replace(" ","_"))
os.rename(infile_F, infile_Ftmp)
os.rename(infile_R, infile_Rtmp)
infile_F = infile_Ftmp
infile_R = infile_Rtmp
libs[k] = [infile_F, infile_R]
# We need this info for later..
coverageFiles.append("%s"%(os.path.join(workdir, k, "%s.coverage.json"%(k)) ) )
except:
pass
configfile = dict(libraries=libs, findcsr = dict( minlen = str(sparams.get('minlen',settings.MIN_SEQ_LEN) ) , msim = str(sparams.get('sim', settings.MIN_SEQ_SIM) ) ) )
with open(os.path.join(workdir, "config.yaml"), "w") as ymout:
yaml.dump(configfile, ymout)
with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout:
if sparams.get("samples", "multi") != "multi" and len(libs) <= 2:
coverageFiles = [os.path.join(workdir, "csr", "combined.coverage.json") ]
yaml.dump(dict(coverage=coverageFiles), ymout)
# This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts
if sparams.get("samples", "multi") == "multi":
if(sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
if(sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single"%( os.path.join(workdir, "config.yaml") , "seanome.db3" , settings.WORK_THREADS))
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(), count = settings.REDIS_THREADS, namespace = 'SeanomeWorkers')
rdis_semaphore.acquire()
catdir = os.path.join(workdir, "concat_trimmed")
if not jobj.interrupt:
os.system("""bash primary_script.sh >> log.out 2>> log.err""")
for c in getSizes(workdir):
os.system("""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """%dict(dbase = os.path.join(workdir, "csr", "seanome.db3"), cnt = c, outprefix = os.path.join(catdir, "%s_concat"%(c)) ) )
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
else:
os.system("""bash sample_runner_script_1.sh >> log.out 2>> log.err """)
if sparams.get("samples", "multi") != "multi":
os.system("""bash sample_runner_script_2.sh >> log.out 2>> log.err """)
jobj = job.objects.get(pk = jid)
jobj.stage = STAGES_REVERSE['done-s1']
jobj.save()
rdis_semaphore.release()
return True
def seanomeGO(workdir, jid, idmap):
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['running']
sparams = json.loads(jobj.scriptparams)
jobj.save()
os.system("""chmod -R 775 %s""" % (workdir))
setupUserEnv(workdir)
close_old_connections()
tasks = []
libs = {}
coverageFiles = []
for k, row in idmap.iteritems():
infile_F = os.path.join(workdir, row[0])
infile_R = os.path.join(workdir, row[1])
try:
infile_Ftmp = os.path.join(workdir, row[0].replace(" ", "_"))
infile_Rtmp = os.path.join(workdir, row[1].replace(" ", "_"))
os.rename(infile_F, infile_Ftmp)
os.rename(infile_R, infile_Rtmp)
infile_F = infile_Ftmp
infile_R = infile_Rtmp
libs[k] = [infile_F, infile_R]
# We need this info for later..
coverageFiles.append(
"%s" % (os.path.join(workdir, k, "%s.coverage.json" % (k))))
except:
pass
configfile = dict(
libraries=libs,
findcsr=dict(minlen=str(sparams.get('minlen', settings.MIN_SEQ_LEN)),
msim=str(sparams.get('sim', settings.MIN_SEQ_SIM))))
with open(os.path.join(workdir, "config.yaml"), "w") as ymout:
yaml.dump(configfile, ymout)
with open(os.path.join(workdir, "coverageLocation.yaml"), "w") as ymout:
if sparams.get("samples", "multi") != "multi" and len(libs) <= 2:
coverageFiles = [
os.path.join(workdir, "csr", "combined.coverage.json")
]
yaml.dump(dict(coverage=coverageFiles), ymout)
# This will generate --ALL-- scripts that are requied to run the seanome pipeline. instead of using the primary script, we will utilize the intermediate scripts
if sparams.get("samples", "multi") == "multi":
if (sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 multiple" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s multiple" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
if (sparams.get('repeatmask', 0) == 1):
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 single" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
else:
os.system("SeanomeWrapper.py -c %s -d %s -t %s -j 1 -s single" %
(os.path.join(workdir, "config.yaml"), "seanome.db3",
settings.WORK_THREADS))
# used in an attempt to limit how many threads of processing a task spins up at any given time.
rdis_semaphore = Semaphore(Redis(),
count=settings.REDIS_THREADS,
namespace='SeanomeWorkers')
rdis_semaphore.acquire()
catdir = os.path.join(workdir, "concat_trimmed")
if not jobj.interrupt:
os.system("""bash primary_script.sh >> log.out 2>> log.err""")
for c in getSizes(workdir):
os.system(
"""combineAlignment.py -d %(dbase)s -c %(cnt)s -o %(outprefix)s """
% dict(dbase=os.path.join(workdir, "csr", "seanome.db3"),
cnt=c,
outprefix=os.path.join(catdir, "%s_concat" % (c))))
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s2']
jobj.save()
else:
os.system(
"""bash sample_runner_script_1.sh >> log.out 2>> log.err """)
if sparams.get("samples", "multi") != "multi":
os.system(
"""bash sample_runner_script_2.sh >> log.out 2>> log.err """)
jobj = job.objects.get(pk=jid)
jobj.stage = STAGES_REVERSE['done-s1']
jobj.save()
rdis_semaphore.release()
return True
