def process_queue(ofs, stride):
possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
selected_jobs = possible_spacer_jobs
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if(len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job = j,**spacer_info))
j.computed_spacers = True
Session.add(j)
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.id
print j.sequence
raise e
def commence_file_io(job_id):
print "DEBUG?? ", "{0}".format( cfront_settings.get("debug_mode",False))
if v: print "commencing file io"
job = Session.query(Job).get(job_id)
Session.add(job)
if len(job.good_spacers) == 0 or job.genome_name != 'hg19':
job.files_failed = True
job.date_failed = datetime.datetime.utcnow()
return
write_f4(job_id)
write_f5_f6(job_id)
if len([f for f in job.files if f["ready"] == False]) == 0:
job.files_ready = True
job.date_completed = datetime.datetime.utcnow()
else:
job.files_failed = True
try:
if job.email_complete:
if (not cfront_settings.get("debug_mode",False)) and cfront_settings.get("mails_on_completion",False):
mail.mail_completed_job(None, job)
except mail.MailError as m:
print "writing mail on complete failed for job {0}".format(job.id)
def process_queue(ofs, stride):
# start a transaction to compute spacers for any jobs which lack them
# NOTE this is the second place in our code where spacers can be created
# this should be fixed as we don't do the same checks here that we do in
# the first -- at views_ajax/post_new.
#
# here we protect against the case where no spacers pass but don't do
# any checking of the input query.
possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
selected_jobs = [j for j in possible_spacer_jobs if j.id % stride == ofs and j.genome_name != "rn5"]
for j in selected_jobs[:100]:
with transaction.manager:
try:
print "computing spacers {0}".format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.sequence
def add_jobs(data_tsv):
cols = ['id', 'sequence', 'date_submitted', 'genome', 'name', 'email', 'date_completed', 'chr', 'start', 'strand', 'computing_spacers', 'computed_spacers', 'files_computing', 'files_ready', 'email_complete', 'key']
rows = []
with open(data_tsv) as f :
for l in f:
rows.append(dict( zip(cols,[e.strip() for e in l.split('\t')]) ))
with transaction.manager:
for r in rows:
sequence = r["sequence"].upper()
sequence = re.sub("\s","",sequence)
if re.compile("[^AGTC]").search(sequence) is not None:
continue
j = Job(key = r['key'],
name = r["name"],
email = r["email"],
chr = r["chr"],
start = r["start"],
strand = r["strand"],
sequence = sequence,
genome = r["genome"],
email_complete=True if r["email_complete"] else False,
date_submitted = datetime.datetime.utcnow())
Session.add(j)
def commence_file_io(job_id):
print "DEBUG?? ", "{0}".format(cfront_settings.get("debug_mode", False))
if v: print "commencing file io"
job = Session.query(Job).get(job_id)
Session.add(job)
if len(job.good_spacers) == 0 or job.genome_name != 'hg19':
job.files_failed = True
job.date_failed = datetime.datetime.utcnow()
return
write_f4(job_id)
write_f5_f6(job_id)
if len([f for f in job.files if f["ready"] == False]) == 0:
job.files_ready = True
job.date_completed = datetime.datetime.utcnow()
else:
job.files_failed = True
try:
if job.email_complete:
if (not cfront_settings.get("debug_mode",
False)) and cfront_settings.get(
"mails_on_completion", False):
mail.mail_completed_job(None, job)
except mail.MailError as m:
print "writing mail on complete failed for job {0}".format(job.id)
def process_queue(ofs, stride):
# start a transaction to compute spacers for any jobs which lack them
# NOTE this is the second place in our code where spacers can be created
# this should be fixed as we don't do the same checks here that we do in
# the first -- at views_ajax/post_new.
#
#here we protect against the case where no spacers pass but don't do
# any checking of the input query.
possible_spacer_jobs = Session.query(Job).filter(
Job.computed_spacers == False).all()
selected_jobs = [
j for j in possible_spacer_jobs
if j.id % stride == ofs and j.genome_name != "rn5"
]
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(
j.id)
print j.sequence
def main(global_config, **settings):
"""
This function returns a Pyramid WSGI application.
"""
import inspect
set_cfront_settings(settings)
set_genomes_settings(settings)
engine = engine_from_config(settings, 'sqlalchemy.')
Session.configure(bind=engine)
Base.metadata.bind = engine
config = Configurator(settings=settings)
#static routes
config.add_static_view('/css', 'static/css', cache_max_age=3600)
config.add_static_view('/js', 'static/js', cache_max_age=3600)
config.add_static_view('/img', 'static/img', cache_max_age=3600)
config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
config.add_static_view('/files', 'static/files', cache_max_age=3600)
#page routes
config.add_route('about', '/about', factory=PageResourceFactory)
config.add_route('batch', '/batch/{batch_key}', factory = BatchResourceFactory)
config.add_route('submit', '/', factory=PageResourceFactory)
config.add_route('job', '/job/{job_key}', factory=PageResourceFactory)
config.add_route('readonly', '/readonly',factory=PageResourceFactory)
config.add_route('readout', '/guides/{job_key}',factory=PageResourceFactory)
config.add_route('nickase', '/nick/{job_key}',factory=PageResourceFactory)
config.add_route('downloads', '/downloads/{job_key}',factory=PageResourceFactory)
#export routes
config.add_route('gb_all_nicks', '/export/nicks_gb/{job_key}',factory=JobResourceFactory)
config.add_route('gb_one_nick', '/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',factory=JobResourceFactory)
config.add_route('csv_one_spacer', '/export/spacer_csv/{job_key}/{spacerid}',factory=JobResourceFactory)
config.add_route('csv_all_guides', '/export/csv_all_guides/{job_key}',factory=JobResourceFactory)
config.add_route('gb_all_guides', '/export/guides_gb/{job_key}',factory=JobResourceFactory)
#ajax routes
config.add_route('job_check_spacers','/j/check_spacers/{job_key}',factory=JobResourceFactory)
config.add_route('job_post_new', '/j/post_new')
config.add_route('jobs_from_fasta', '/j/from_fasta')
config.add_route('job_from_spacers', '/j/from_spacers')
config.add_route('job_retrieve_spacers', '/j/retrieve_spacers/{job_key}',factory=JobResourceFactory)
config.add_route('job_email_complete', '/j/email_complete/{job_key}',factory=JobResourceFactory)
config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
config.add_route('spacer_retrieve_regions', '/s/retrieve_regions/{spacer_id}')
#rest routes
config.add_route('job_rest', '/r/job/{job_key}',factory=JobResourceFactory)
config.add_route('batch_rest', '/r/batch/{batch_key}',factory=BatchResourceFactory)
config.add_route('spacer_rest', '/r/spacer/{spacer_id}')
config.add_route('maintainance','/sorry')
config.scan()
return config.make_wsgi_app()
def estimated_waiting_time(self):
deltatime = timedelta(days = 7)
t1 = datetime.now() - deltatime
possible_hit_jobs_total = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid == None)\
.order_by('-id')\
.limit(100)\
.all()
ids = [i.id for i in possible_hit_jobs_total]
if self.id not in ids:
return 24*60*60
else:
m1 = ids.index(self.id)
m2 = len(possible_hit_jobs_total)
N = 12
if (N - 1) <= m1:
if m1 >= 50:
p1 = 0.0
p2 = 0.0
else:
p1 = 1.0/2.0*round(sum([scipy.misc.comb(m1,ii) for ii in range(0,N-1 + 1)]))/scipy.power(2,m1)
p2 = 0.0
else:
if (m2-m1) >= 50:
p1 = 0.5
p2 = 0.0
else:
p1 = 0.5
p2 = 1.0/2.0*round(sum([scipy.misc.comb(m2-m1,ii) for ii in range(0,N-1-m1 + 1)]))/scipy.power(2,m2-m1)
p = p1 + p2
print m1, m2, p1, p2, p
return int(ceil((self.n_spacers - self.n_completed_spacers)/2.0) * ceil(1.0/p * 60.0))
def top_hits(self):
from cfront.models import Hit
return [
e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
Spacer.id == self.id).order_by(desc(Hit.score)).limit(
50).all()
]
def write_f5_f6(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
f5p = job.f5
f6p = job.f6
gene = os.path.join(job.safe_name)
if os.path.isfile(f5p):
os.remove(f5p)
if os.path.isfile(f6p):
os.remove(f6p)
if v: print "writing f5 at {0}".format(f5p)
if v: print "writing f6 at {0}".format(f6p)
cmd = ("R -f "+\
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
def write_f5_f6(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
f5p = job.f5
f6p = job.f6
gene= os.path.join(job.safe_name)
if os.path.isfile(f5p):
os.remove(f5p)
if os.path.isfile(f6p):
os.remove(f6p)
if v: print "writing f5 at {0}".format(f5p)
if v: print "writing f6 at {0}".format(f6p)
cmd = ("R -f "+\
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
def write_f4(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
if os.path.isfile(f4p):
os.remove(f4p)
if v: print "writing f4 at {0}".format(f4p)
with open(f4p,"w") as f:
f.write(" ".join(["SPACER","ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE", "MISMATCH", "MISMATCH_POS", "SCORE", "GENE"])+"\n")
for spacer in job.good_spacers:
ot_count=1
for hit in sorted(spacer.hits, key=lambda x:-1* x.score):
spacerid = "spacer_{0}".format(spacer.id)
ontarget = 1 if hit.ontarget else 0
if ontarget:
hitid = "{0}".format(spacerid)
else:
hitid = "{0}_off_{1}".format(spacerid, ot_count)
ot_count+=1
chr=hit.chr[3:]
pos=hit.start
strand="+" if hit.strand == 1 else "-"
sequence=hit.sequence
mismatch=hit.n_mismatches
mismatch_pos= "NA" if mismatch == 0 \
else ":".join( ["{0}".format(i) for i in range(20)
if spacer.sequence[i] != hit.sequence[i]])
score = "{0:.1f}".format(hit.score)
gene = hit.gene
row = " ".join("{0}".format(e) for e in [spacerid, ontarget, hitid, chr, pos, strand, sequence, mismatch,
mismatch_pos, score, gene])
f.write(row+"\n")
if v: print "DONE"
if v: print
def write_f4(job_id):
job = Session.query(Job).get(job_id)
f4p = job.f4
if os.path.isfile(f4p):
os.remove(f4p)
if v: print "writing f4 at {0}".format(f4p)
with open(f4p, "w") as f:
f.write(" ".join([
"SPACER", "ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE",
"MISMATCH", "MISMATCH_POS", "SCORE", "GENE"
]) + "\n")
for spacer in job.good_spacers:
ot_count = 1
for hit in sorted(spacer.hits, key=lambda x: -1 * x.score):
spacerid = "spacer_{0}".format(spacer.id)
ontarget = 1 if hit.ontarget else 0
if ontarget:
hitid = "{0}".format(spacerid)
else:
hitid = "{0}_off_{1}".format(spacerid, ot_count)
ot_count += 1
chr = hit.chr[3:]
pos = hit.start
strand = "+" if hit.strand == 1 else "-"
sequence = hit.sequence
mismatch = hit.n_mismatches
mismatch_pos= "NA" if mismatch == 0 \
else ":".join( ["{0}".format(i) for i in range(20)
if spacer.sequence[i] != hit.sequence[i]])
score = "{0:.1f}".format(hit.score)
gene = hit.gene
row = " ".join("{0}".format(e) for e in [
spacerid, ontarget, hitid, chr, pos, strand, sequence,
mismatch, mismatch_pos, score, gene
])
f.write(row + "\n")
if v: print "DONE"
if v: print
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
j.failed = True
Session.add(j)
print j.id
print j.sequence
raise e
#change on 3/14/2016
deltatime = timedelta(days = 7)
t1 = datetime.now() - deltatime
possible_hit_jobs = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid == None)\
.filter(Job.computed_spacers == True)\
.order_by('-id')\
.limit(100)\
.all()
possible_hit_batch_jobs = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.date_completed == None)\
.filter(Job.batchid != None)\
.filter(Job.computed_spacers == True)\
.order_by('-id')\
.limit(100)\
.all()
selected_hit_jobs = possible_hit_jobs + possible_hit_batch_jobs
print "computing spacers {0}".format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if (len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.sequence
# starts a transaction to compute hits for any spacer which has not
# "computed_hits" or "computing_hits"
possible_hit_jobs = Session.query(Job).join(Spacer).filter(Spacer.score == None).filter(Job.failed == False).all()
selected_hit_jobs = [j for j in possible_hit_jobs if j.id % stride == ofs and j.genome_name != "rn5"]
if len(selected_hit_jobs) > 0:
batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
# sorts jobs to process recent submissions and non-batch jobs first
def priority(j):
# return 1 * j.id
from random import random
f = 1 if random() > 0.5 else -1
if j.key == "7956546276189290":
return -100000
if j.batch is not None:
return 100000000
else:
def genic_hits(self):
from cfront.models import Hit
return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).filter(Hit.gene != None).all()]
def top_hits(self):
from cfront.models import Hit
return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).order_by(desc(Hit.score)).limit(50).all()]
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
env = bootstrap(sys.argv[1])
while 1:
j = Session.query(Job)\
.filter(Job.files_failed == False)\
.filter(Job.files_ready == False)\
.join(Spacer)\
.group_by(Job.id)\
.having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
.order_by(Job.id.desc()).first()
if j:
with transaction.manager:
commence_file_io(j.id)
time.sleep(2)
"{0}/make_graphical_output.R --slave --vanilla --args "+\
"{1} {2} {3} {4}")\
.format(CDSCRIPTS,f4p,f5p,gene,f6p)
if v: print "running cmd:"
if v: print " {0}".format(cmd)
prc = spc.Popen(cmd, shell=True)
out = prc.communicate()
print "outval: {0}".format(out)
if v: print "DONE"
if v: print
import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
env = bootstrap(sys.argv[1])
while 1:
j = Session.query(Job)\
.filter(Job.files_failed == False)\
.filter(Job.files_ready == False)\
.join(Spacer)\
.group_by(Job.id)\
.having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
.order_by(Job.id.desc()).first()
if j:
with transaction.manager:
commence_file_io(j.id)
time.sleep(2)
def get_batch_by_key(batch_key):
b = Session.query(Batch).filter(Batch.key == batch_key).first()
if b is None:
from cfront.models import BatchNOTFOUND
raise BatchNOTFOUND("batch not found {0}".format(batch_key), None)
return b
def main(global_config, **settings):
"""
This function returns a Pyramid WSGI application.
"""
import inspect
set_cfront_settings(settings)
set_genomes_settings(settings)
engine = engine_from_config(settings, 'sqlalchemy.')
Session.configure(bind=engine)
Base.metadata.bind = engine
config = Configurator(settings=settings)
config.include('pyramid_mako')
#static routes
config.add_static_view('/v2/css', 'static/css', cache_max_age=3600)
config.add_static_view('/v2/js', 'static/js', cache_max_age=3600)
config.add_static_view('/img', 'static/img', cache_max_age=3600)
config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
config.add_static_view('/files', 'static/files', cache_max_age=3600)
#page routes
config.add_route('splash_v2', '/v2', factory=PageResourceFactory)
config.add_route('gene_results_v2', '/v2/{assembly}/{geneid}/gene_results', factory=PageResourceFactory)
config.add_route('submit_v2', '/v2/{assembly}/submit', factory=PageResourceFactory)
config.add_route('genes_autocomplete_array', '/v2/{assembly}/gene_names.json')
config.add_route('genes_autocomplete_info', '/v2/{assembly}/genes_info.json')
config.add_route('gene_genbank', '/v2/{assembly}/{geneid}/gene.gb')
config.add_route('gene_genbank_spacer', '/v2/{assembly}/{geneid}/{tool}/{guide_sequence}/guide.gb')
config.add_route('gene_genbank_all_spacers', '/v2/{assembly}/{geneid}/top_spacers.gb')
config.add_route('gene_genbank_selected_spacers', '/v2/{assembly}/{geneid}/selected_spacers.gb')
config.add_route('gene_fasta', '/v2/{assembly}/{geneid}/gene.fa')
config.add_route('gene_genbank_json', '/v2/{assembly}/{geneid}/genbank.json')
config.add_route('gene_sequence', '/v2/{assembly}/{geneid}/sequence.json')
config.add_route('job_data','/v2/{assembly}/{geneid}/job.json')
config.add_route('email_all_genbank', '/v2/{assembly}/{geneid}/{email}/email_all_genbank')
config.add_route('about', '/about', factory=PageResourceFactory)
config.add_route('batch', '/v1/batch/{batch_key}', factory = BatchResourceFactory)
config.add_route('submit', '/', factory=PageResourceFactory)
config.add_route('job', '/v1/job/{job_key}', factory=PageResourceFactory)
config.add_route('readonly', '/readonly',factory=PageResourceFactory)
config.add_route('readout', '/v1/guides/{job_key}',factory=PageResourceFactory)
config.add_route('nickase', '/v1/nick/{job_key}',factory=PageResourceFactory)
config.add_route('downloads', '/v1/downloads/{job_key}',factory=PageResourceFactory)
#export routes
config.add_route('gb_all_nicks', '/v1/export/nicks_gb/{job_key}',factory=JobResourceFactory)
config.add_route('gb_one_nick', '/v1/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',factory=JobResourceFactory)
config.add_route('csv_one_spacer', '/v1/export/spacer_csv/{job_key}/{spacerid}',factory=JobResourceFactory)
config.add_route('csv_all_guides', '/v1/export/csv_all_guides/{job_key}',factory=JobResourceFactory)
config.add_route('gb_all_guides', '/v1/export/guides_gb/{job_key}',factory=JobResourceFactory)
#v2 ajax routes
config.add_route('v2_query_gene', '/v2/{assembly}/{geneid}/{email}/query_gene')
#ajax routes
config.add_route('job_check_spacers','/j/check_spacers/{job_key}',factory=JobResourceFactory)
config.add_route('job_post_new', '/j/post_new')
config.add_route('jobs_from_fasta', '/j/from_fasta')
config.add_route('job_from_spacers', '/j/from_spacers')
config.add_route('job_retrieve_spacers', '/j/retrieve_spacers/{job_key}',factory=JobResourceFactory)
config.add_route('job_email_complete', '/j/email_complete/{job_key}',factory=JobResourceFactory)
config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
config.add_route('spacer_retrieve_regions', '/s/retrieve_regions/{spacer_id}')
#rest routes
config.add_route('job_rest', '/r/job/{job_key}',factory=JobResourceFactory)
config.add_route('batch_rest', '/r/batch/{batch_key}',factory=BatchResourceFactory)
config.add_route('spacer_rest', '/r/spacer/{spacer_id}')
config.add_route('maintainance','/sorry')
config.scan()
return config.make_wsgi_app()
def main(global_config, **settings):
"""
This function returns a Pyramid WSGI application.
"""
import inspect
set_cfront_settings(settings)
set_genomes_settings(settings)
engine = engine_from_config(settings, 'sqlalchemy.')
Session.configure(bind=engine)
Base.metadata.bind = engine
config = Configurator(settings=settings)
#static routes
config.add_static_view('/css', 'static/css', cache_max_age=3600)
config.add_static_view('/js', 'static/js', cache_max_age=3600)
config.add_static_view('/img', 'static/img', cache_max_age=3600)
config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
config.add_static_view('/files', 'static/files', cache_max_age=3600)
#page routes
config.add_route('about', '/about', factory=PageResourceFactory)
config.add_route('batch',
'/batch/{batch_key}',
factory=BatchResourceFactory)
config.add_route('submit', '/', factory=PageResourceFactory)
config.add_route('job', '/job/{job_key}', factory=PageResourceFactory)
config.add_route('readonly', '/readonly', factory=PageResourceFactory)
config.add_route('readout',
'/guides/{job_key}',
factory=PageResourceFactory)
config.add_route('nickase', '/nick/{job_key}', factory=PageResourceFactory)
config.add_route('downloads',
'/downloads/{job_key}',
factory=PageResourceFactory)
#export routes
config.add_route('gb_all_nicks',
'/export/nicks_gb/{job_key}',
factory=JobResourceFactory)
config.add_route('gb_one_nick',
'/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',
factory=JobResourceFactory)
config.add_route('csv_one_spacer',
'/export/spacer_csv/{job_key}/{spacerid}',
factory=JobResourceFactory)
config.add_route('csv_all_guides',
'/export/csv_all_guides/{job_key}',
factory=JobResourceFactory)
config.add_route('gb_all_guides',
'/export/guides_gb/{job_key}',
factory=JobResourceFactory)
#ajax routes
config.add_route('job_check_spacers',
'/j/check_spacers/{job_key}',
factory=JobResourceFactory)
config.add_route('job_post_new', '/j/post_new')
config.add_route('jobs_from_fasta', '/j/from_fasta')
config.add_route('job_from_spacers', '/j/from_spacers')
config.add_route('job_retrieve_spacers',
'/j/retrieve_spacers/{job_key}',
factory=JobResourceFactory)
config.add_route('job_email_complete',
'/j/email_complete/{job_key}',
factory=JobResourceFactory)
config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
config.add_route('spacer_retrieve_regions',
'/s/retrieve_regions/{spacer_id}')
#rest routes
config.add_route('job_rest',
'/r/job/{job_key}',
factory=JobResourceFactory)
config.add_route('batch_rest',
'/r/batch/{batch_key}',
factory=BatchResourceFactory)
config.add_route('spacer_rest', '/r/spacer/{spacer_id}')
config.add_route('maintainance', '/sorry')
config.scan()
return config.make_wsgi_app()
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job=j, **spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(
j.id)
print j.sequence
#starts a transaction to compute hits for any spacer which has not
# "computed_hits" or "computing_hits"
possible_hit_jobs = Session.query(Job)\
.join(Spacer)\
.filter(Spacer.score == None)\
.filter(Job.failed == False)\
.all()
selected_hit_jobs = [
j for j in possible_hit_jobs
if j.id % stride == ofs and j.genome_name != "rn5"
]
if len(selected_hit_jobs) > 0:
batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
#sorts jobs to process recent submissions and non-batch jobs first
def priority(j):
#return 1 * j.id
from random import random
f = 1 if random() > .5 else -1
def get_job_by_key(job_key):
j = Session.query(Job).filter(Job.key == job_key).first()
if j is None:
from cfront.models import JobNOTFOUND
raise JobNOTFOUND("job not found {0}".format(job_key), None)
return j
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.id
print j.sequence
raise e
possible_hit_jobs = Session.query(Job)\
.join(Spacer)\
.filter(Spacer.score == None)\
.filter(Job.failed == False)\
.all()
selected_hit_jobs = possible_hit_jobs
procs = []
max_procs = 8
manager = Manager()
jobs_q = JoinableQueue()
r = redis.Redis()
dcount = 0
while(1):
def get_job_by_key(job_key):
j = Session.query(Job).filter(Job.key == job_key).first()
if j is None:
from cfront.models import JobNOTFOUND
raise JobNOTFOUND("job not found {0}".format(job_key), None)
return j
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("inifile",
help="inifile to configure the server")
args = parser.parse_args()
#starts up the paster env
env = bootstrap(args.inifile)
jobpath = cfront_settings["jobs_directory"]
#for d in os.listdir(jobpath):
# print "removing directory: {0}".format(d)
# shutil.rmtree(os.path.join(jobpath,d))
# print os.path.join(jobpath,d)
with transaction.manager:
for j in Session.query(Job).order_by(Job.id).all():
if j.genome_name == "hg19" :
continue
else:
for s in j.spacers:
Session.delete(s)
print "deleting spacers for job {0}".format(j.id)
j.computed_spacers = False
j.files_ready=False
j.date_completed = None
j.files_failed =False
j.failed = False
def genic_hits(self):
from cfront.models import Hit
return [
e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
Spacer.id == self.id).filter(Hit.gene != None).all()
]
def process_queue(ofs, stride):
deltatime = timedelta(days = 7)
t1 = datetime.now() - deltatime
possible_spacer_jobs = Session.query(Job)\
.filter(Job.date_submitted > t1)\
.filter(Job.failed == False)\
.filter(Job.computed_spacers == False)\
.all()
selected_jobs = possible_spacer_jobs
for j in selected_jobs[:100]:
with transaction.manager:
try:
print 'computing spacers {0}'.format(j.id)
spacer_infos = webserver_db.compute_spacers(j.sequence)
if(len(spacer_infos)) == 0:
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job = j,**spacer_info))
j.computed_spacers = True
Session.add(j)
print j.id
except JobERR, e:
if e.message == Job.NOSPACERS:
j.failed = True
j.computed_spacers = True
Session.add(j)
print "No spacers in JOB ID: {0}".format(j.id)
elif e.message == Job.ERR_INVALID_CHARACTERS:
j.failed = True
j.computed_spacers = True
Session.add(j)
print e.message
else:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
j.failed = True
Session.add(j)
print j.id
print j.sequence
raise e
raise JobERR(Job.NOSPACERS, j)
for spacer_info in spacer_infos:
Session.add(Spacer(job = j,**spacer_info))
j.computed_spacers = True
Session.add(j)
except JobERR, e:
print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
print j.sequence
#starts a transaction to compute hits for any spacer which has not
# "computed_hits" or "computing_hits"
possible_hit_jobs = Session.query(Job)\
.join(Spacer)\
.filter(Spacer.score == None)\
.filter(Job.failed == False)\
.all()
selected_hit_jobs = [j for j in possible_hit_jobs if j.id % stride == ofs and j.genome_name != "rn5"]
if len(selected_hit_jobs) > 0:
batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
#sorts jobs to process recent submissions and non-batch jobs first
def priority(j):
#return 1 * j.id
from random import random
f = 1 if random() > .5 else -1
if j.key == "7956546276189290" : return -100000
if j.batch is not None: return 100000000
else: return f* j.id
import transaction
import os, shutil
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("inifile", help="inifile to configure the server")
args = parser.parse_args()
#starts up the paster env
env = bootstrap(args.inifile)
jobpath = cfront_settings["jobs_directory"]
#for d in os.listdir(jobpath):
# print "removing directory: {0}".format(d)
# shutil.rmtree(os.path.join(jobpath,d))
# print os.path.join(jobpath,d)
with transaction.manager:
for j in Session.query(Job).order_by(Job.id).all():
if j.genome_name == "hg19":
continue
else:
for s in j.spacers:
Session.delete(s)
print "deleting spacers for job {0}".format(j.id)
j.computed_spacers = False
j.files_ready = False
j.date_completed = None
j.files_failed = False
j.failed = False