def estimated_waiting_time(self):
deltatime = timedelta(days = 7)
t1 = datetime.now() - deltatime
possible_hit_jobs_total = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid == None)\
.order_by('-id')\
.limit(100)\
.all()
ids = [i.id for i in possible_hit_jobs_total]
if self.id not in ids:
return 24*60*60
else:
m1 = ids.index(self.id)
m2 = len(possible_hit_jobs_total)
N = 12
if (N - 1) <= m1:
if m1 >= 50:
p1 = 0.0
p2 = 0.0
else:
p1 = 1.0/2.0*round(sum([scipy.misc.comb(m1,ii) for ii in range(0,N-1 + 1)]))/scipy.power(2,m1)
p2 = 0.0
else:
if (m2-m1) >= 50:
p1 = 0.5
p2 = 0.0
else:
p1 = 0.5
p2 = 1.0/2.0*round(sum([scipy.misc.comb(m2-m1,ii) for ii in range(0,N-1-m1 + 1)]))/scipy.power(2,m2-m1)
p = p1 + p2
print m1, m2, p1, p2, p
return int(ceil((self.n_spacers - self.n_completed_spacers)/2.0) * ceil(1.0/p * 60.0))
def process_queue(ofs, stride):
possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
selected_jobs = possible_spacer_jobs
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if(len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job = j,**spacer_info))
j.computed_spacers = True
Session.add(j)
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
j.failed = True
j.computed_spacers = True
Session.add(j)
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
j.failed = True
j.computed_spacers = True
Session.add(j)
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
j.failed = True
Session.add(j)
print j.id
print j.sequence
raise e
def write_f5_f6(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
f5p = job.f5
f6p = job.f6
gene = os.path.join(job.safe_name)
if os.path.isfile(f5p):
os.remove(f5p)
if os.path.isfile(f6p):
os.remove(f6p)
if v: print "writing f5 at {0}".format(f5p)
if v: print "writing f6 at {0}".format(f6p)
cmd = ("R -f "+\
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
def top_hits(self):
from cfront.models import Hit
return [
e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
Spacer.id == self.id).order_by(desc(Hit.score)).limit(
50).all()
]
def commence_file_io(job_id):
print "DEBUG?? ", "{0}".format(cfront_settings.get("debug_mode", False))
if v: print "commencing file io"
job = Session.query(Job).get(job_id)
Session.add(job)
if len(job.good_spacers) == 0 or job.genome_name != 'hg19':
job.files_failed = True
job.date_failed = datetime.datetime.utcnow()
return
write_f4(job_id)
write_f5_f6(job_id)
if len([f for f in job.files if f["ready"] == False]) == 0:
job.files_ready = True
job.date_completed = datetime.datetime.utcnow()
else:
job.files_failed = True
try:
if job.email_complete:
if (not cfront_settings.get("debug_mode",
False)) and cfront_settings.get(
"mails_on_completion", False):
mail.mail_completed_job(None, job)
except mail.MailError as m:
print "writing mail on complete failed for job {0}".format(job.id)
def process_queue(ofs, stride):
possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
selected_jobs = possible_spacer_jobs
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if(len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job = j,**spacer_info))
j.computed_spacers = True
Session.add(j)
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.id
print j.sequence
raise e
def process_queue(ofs, stride):
# start a transaction to compute spacers for any jobs which lack them
# NOTE this is the second place in our code where spacers can be created
# this should be fixed as we don't do the same checks here that we do in
# the first -- at views_ajax/post_new.
#
# here we protect against the case where no spacers pass but don't do
# any checking of the input query.
possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
selected_jobs = [j for j in possible_spacer_jobs if j.id % stride == ofs and j.genome_name != "rn5"]
for j in selected_jobs[:100]:
with transaction.manager:
try:
print "computing spacers {0}".format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.sequence
def process_queue(ofs, stride):
# start a transaction to compute spacers for any jobs which lack them
# NOTE this is the second place in our code where spacers can be created
# this should be fixed as we don't do the same checks here that we do in
# the first -- at views_ajax/post_new.
#
#here we protect against the case where no spacers pass but don't do
# any checking of the input query.
possible_spacer_jobs = Session.query(Job).filter(
Job.computed_spacers == False).all()
selected_jobs = [
j for j in possible_spacer_jobs
if j.id % stride == ofs and j.genome_name != "rn5"
]
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(
j.id)
print j.sequence
def write_f4(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
if os.path.isfile(f4p):
os.remove(f4p)
if v: print "writing f4 at {0}".format(f4p)
with open(f4p,"w") as f:
f.write(" ".join(["SPACER","ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE", "MISMATCH", "MISMATCH_POS", "SCORE", "GENE"])+"\n")
for spacer in job.good_spacers:
ot_count=1
for hit in sorted(spacer.hits, key=lambda x:-1* x.score):
spacerid = "spacer_{0}".format(spacer.id)
ontarget = 1 if hit.ontarget else 0
if ontarget:
hitid = "{0}".format(spacerid)
else:
hitid = "{0}_off_{1}".format(spacerid, ot_count)
ot_count+=1
chr=hit.chr[3:]
pos=hit.start
strand="+" if hit.strand == 1 else "-"
sequence=hit.sequence
mismatch=hit.n_mismatches
mismatch_pos= "NA" if mismatch == 0 \
else ":".join( ["{0}".format(i) for i in range(20)
if spacer.sequence[i] != hit.sequence[i]])
score = "{0:.1f}".format(hit.score)
gene = hit.gene
row = " ".join("{0}".format(e) for e in [spacerid, ontarget, hitid, chr, pos, strand, sequence, mismatch,
mismatch_pos, score, gene])
f.write(row+"\n")
if v: print "DONE"
if v: print
def write_f5_f6(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
f5p = job.f5
f6p = job.f6
gene= os.path.join(job.safe_name)
if os.path.isfile(f5p):
os.remove(f5p)
if os.path.isfile(f6p):
os.remove(f6p)
if v: print "writing f5 at {0}".format(f5p)
if v: print "writing f6 at {0}".format(f6p)
cmd = ("R -f "+\
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
def commence_file_io(job_id):
print "DEBUG?? ", "{0}".format( cfront_settings.get("debug_mode",False))
if v: print "commencing file io"
job = Session.query(Job).get(job_id)
Session.add(job)
if len(job.good_spacers) == 0 or job.genome_name != 'hg19':
job.files_failed = True
job.date_failed = datetime.datetime.utcnow()
return
write_f4(job_id)
write_f5_f6(job_id)
if len([f for f in job.files if f["ready"] == False]) == 0:
job.files_ready = True
job.date_completed = datetime.datetime.utcnow()
else:
job.files_failed = True
try:
if job.email_complete:
if (not cfront_settings.get("debug_mode",False)) and cfront_settings.get("mails_on_completion",False):
mail.mail_completed_job(None, job)
except mail.MailError as m:
print "writing mail on complete failed for job {0}".format(job.id)
def write_f4(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
if os.path.isfile(f4p):
os.remove(f4p)
if v: print "writing f4 at {0}".format(f4p)
with open(f4p, "w") as f:
f.write(" ".join([
"SPACER", "ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE",
"MISMATCH", "MISMATCH_POS", "SCORE", "GENE"
]) + "\n")
for spacer in job.good_spacers:
ot_count = 1
for hit in sorted(spacer.hits, key=lambda x: -1 * x.score):
spacerid = "spacer_{0}".format(spacer.id)
ontarget = 1 if hit.ontarget else 0
if ontarget:
hitid = "{0}".format(spacerid)
else:
hitid = "{0}_off_{1}".format(spacerid, ot_count)
ot_count += 1
chr = hit.chr[3:]
pos = hit.start
strand = "+" if hit.strand == 1 else "-"
sequence = hit.sequence
mismatch = hit.n_mismatches
mismatch_pos= "NA" if mismatch == 0 \
else ":".join( ["{0}".format(i) for i in range(20)
if spacer.sequence[i] != hit.sequence[i]])
score = "{0:.1f}".format(hit.score)
gene = hit.gene
row = " ".join("{0}".format(e) for e in [
spacerid, ontarget, hitid, chr, pos, strand, sequence,
mismatch, mismatch_pos, score, gene
])
f.write(row + "\n")
if v: print "DONE"
if v: print
def get_job_by_key(job_key):
j = Session.query(Job).filter(Job.key == job_key).first()
if j is None:
from cfront.models import JobNOTFOUND
raise JobNOTFOUND("job not found {0}".format(job_key), None)
return j
def get_job_by_key(job_key):
j = Session.query(Job).filter(Job.key == job_key).first()
if j is None:
from cfront.models import JobNOTFOUND
raise JobNOTFOUND("job not found {0}".format(job_key), None)
return j
print "computing spacers {0}".format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.sequence
# starts a transaction to compute hits for any spacer which has not
# "computed_hits" or "computing_hits"
possible_hit_jobs = Session.query(Job).join(Spacer).filter(Spacer.score == None).filter(Job.failed == False).all()
selected_hit_jobs = [j for j in possible_hit_jobs if j.id % stride == ofs and j.genome_name != "rn5"]
if len(selected_hit_jobs) > 0:
batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
# sorts jobs to process recent submissions and non-batch jobs first
def priority(j):
# return 1 * j.id
from random import random
f = 1 if random() > 0.5 else -1
if j.key == "7956546276189290":
return -100000
if j.batch is not None:
return 100000000
else:
def genic_hits(self):
from cfront.models import Hit
return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).filter(Hit.gene != None).all()]
def top_hits(self):
from cfront.models import Hit
return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).order_by(desc(Hit.score)).limit(50).all()]
def genic_hits(self):
from cfront.models import Hit
return [
e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
Spacer.id == self.id).filter(Hit.gene != None).all()
]
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
env = bootstrap(sys.argv[1])
while 1:
j = Session.query(Job)\
.filter(Job.files_failed == False)\
.filter(Job.files_ready == False)\
.join(Spacer)\
.group_by(Job.id)\
.having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
.order_by(Job.id.desc()).first()
if j:
with transaction.manager:
commence_file_io(j.id)
time.sleep(2)
import transaction
import os, shutil
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("inifile", help="inifile to configure the server")
args = parser.parse_args()
#starts up the paster env
env = bootstrap(args.inifile)
jobpath = cfront_settings["jobs_directory"]
#for d in os.listdir(jobpath):
# print "removing directory: {0}".format(d)
# shutil.rmtree(os.path.join(jobpath,d))
# print os.path.join(jobpath,d)
with transaction.manager:
for j in Session.query(Job).order_by(Job.id).all():
if j.genome_name == "hg19":
continue
else:
for s in j.spacers:
Session.delete(s)
print "deleting spacers for job {0}".format(j.id)
j.computed_spacers = False
j.files_ready = False
j.date_completed = None
j.files_failed = False
j.failed = False
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
env = bootstrap(sys.argv[1])
while 1:
j = Session.query(Job)\
.filter(Job.files_failed == False)\
.filter(Job.files_ready == False)\
.join(Spacer)\
.group_by(Job.id)\
.having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
.order_by(Job.id.desc()).first()
if j:
with transaction.manager:
commence_file_io(j.id)
time.sleep(2)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("inifile",
help="inifile to configure the server")
args = parser.parse_args()
#starts up the paster env
env = bootstrap(args.inifile)
jobpath = cfront_settings["jobs_directory"]
#for d in os.listdir(jobpath):
# print "removing directory: {0}".format(d)
# shutil.rmtree(os.path.join(jobpath,d))
# print os.path.join(jobpath,d)
with transaction.manager:
for j in Session.query(Job).order_by(Job.id).all():
if j.genome_name == "hg19" :
continue
else:
for s in j.spacers:
Session.delete(s)
print "deleting spacers for job {0}".format(j.id)
j.computed_spacers = False
j.files_ready=False
j.date_completed = None
j.files_failed =False
j.failed = False
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.id
print j.sequence
raise e
possible_hit_jobs = Session.query(Job)\
.join(Spacer)\
.filter(Spacer.score == None)\
.filter(Job.failed == False)\
.all()
selected_hit_jobs = possible_hit_jobs
procs = []
max_procs = 8
manager = Manager()
jobs_q = JoinableQueue()
r = redis.Redis()
dcount = 0
while(1):
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(
j.id)
print j.sequence
#starts a transaction to compute hits for any spacer which has not
# "computed_hits" or "computing_hits"
possible_hit_jobs = Session.query(Job)\
.join(Spacer)\
.filter(Spacer.score == None)\
.filter(Job.failed == False)\
.all()
selected_hit_jobs = [
j for j in possible_hit_jobs
if j.id % stride == ofs and j.genome_name != "rn5"
]
if len(selected_hit_jobs) > 0:
batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
#sorts jobs to process recent submissions and non-batch jobs first
def priority(j):
#return 1 * j.id
from random import random
f = 1 if random() > .5 else -1
def get_batch_by_key(batch_key):
b = Session.query(Batch).filter(Batch.key == batch_key).first()
if b is None:
from cfront.models import BatchNOTFOUND
raise BatchNOTFOUND("batch not found {0}".format(batch_key), None)
return b
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
j.failed = True
Session.add(j)
print j.id
print j.sequence
raise e
#change on 3/14/2016
deltatime = timedelta(days = 7)
t1 = datetime.now() - deltatime
possible_hit_jobs = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid == None)\
.filter(Job.computed_spacers == True)\
.order_by('-id')\
.limit(100)\
.all()
possible_hit_batch_jobs = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid != None)\
.filter(Job.computed_spacers == True)\
.order_by('-id')\
.limit(100)\
.all()
selected_hit_jobs = possible_hit_jobs + possible_hit_batch_jobs
