示例#1
0
def process_queue(ofs, stride):
    possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
    selected_jobs = possible_spacer_jobs

    for j in selected_jobs[:100]:
        with transaction.manager:
            try:
                print 'computing spacers {0}'.format(j.id)
                spacer_infos = webserver_db.compute_spacers(j.sequence)
                if(len(spacer_infos)) == 0:
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job = j,**spacer_info))
                j.computed_spacers = True
                Session.add(j)
                print j.id
            except JobERR, e:
                if e.message == Job.NOSPACERS:
                    print "No spacers in JOB ID: {0}".format(j.id)
                elif e.message == Job.ERR_INVALID_CHARACTERS:
                    print e.message
                else:
                    print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                    print j.id
                    print j.sequence
                    raise e
示例#2
0
def commence_file_io(job_id):

    print "DEBUG?? ", "{0}".format( cfront_settings.get("debug_mode",False))

    if v: print "commencing file io"

    job = Session.query(Job).get(job_id)    
    Session.add(job)
    
    if  len(job.good_spacers) == 0 or job.genome_name != 'hg19':
        job.files_failed = True
        job.date_failed = datetime.datetime.utcnow()
        return 

    write_f4(job_id)
    write_f5_f6(job_id)
    
    if len([f for f in job.files if f["ready"] == False]) == 0:
        job.files_ready = True
        job.date_completed = datetime.datetime.utcnow()
    else:
        job.files_failed = True

    
    try:
        if job.email_complete:
            if (not cfront_settings.get("debug_mode",False)) and cfront_settings.get("mails_on_completion",False):
                mail.mail_completed_job(None, job)
    except mail.MailError as m:
        print "writing mail on complete failed for job {0}".format(job.id)
示例#3
0
def process_queue(ofs, stride):
    # start a transaction to compute spacers for any jobs which lack them
    # NOTE this is the second place in our code where spacers can be created
    # this should be fixed as we don't do the same checks here that we do in
    # the first -- at views_ajax/post_new.
    #
    # here we protect against the case where no spacers pass but don't do
    # any checking of the input query.

    possible_spacer_jobs = Session.query(Job).filter(Job.computed_spacers == False).all()
    selected_jobs = [j for j in possible_spacer_jobs if j.id % stride == ofs and j.genome_name != "rn5"]

    for j in selected_jobs[:100]:
        with transaction.manager:
            try:
                print "computing spacers {0}".format(j.id)
                spacer_infos = webserver_db.compute_spacers(j.sequence)
                if (len(spacer_infos)) == 0:
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job=j, **spacer_info))
                j.computed_spacers = True
                Session.add(j)
            except JobERR, e:
                print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                print j.sequence
示例#4
0
def add_jobs(data_tsv):
    cols = ['id', 'sequence', 'date_submitted', 'genome', 'name', 'email', 'date_completed', 'chr', 'start', 'strand', 'computing_spacers', 'computed_spacers', 'files_computing', 'files_ready', 'email_complete', 'key']
    rows = []
    with open(data_tsv) as f :
        for l in f:
            rows.append(dict( zip(cols,[e.strip() for e in  l.split('\t')])  ))
    
    with transaction.manager:
        for r in rows:
            
            sequence = r["sequence"].upper()
            sequence = re.sub("\s","",sequence)
            if re.compile("[^AGTC]").search(sequence) is not None:
                continue

            j = Job(key = r['key'],
                    name = r["name"],
                    email = r["email"],
                    chr = r["chr"],
                    start = r["start"],
                    strand = r["strand"],
                    sequence = sequence,
                    genome = r["genome"],
                    email_complete=True if r["email_complete"] else False,
                    date_submitted = datetime.datetime.utcnow())

            Session.add(j)
示例#5
0
def commence_file_io(job_id):

    print "DEBUG?? ", "{0}".format(cfront_settings.get("debug_mode", False))

    if v: print "commencing file io"

    job = Session.query(Job).get(job_id)
    Session.add(job)

    if len(job.good_spacers) == 0 or job.genome_name != 'hg19':
        job.files_failed = True
        job.date_failed = datetime.datetime.utcnow()
        return

    write_f4(job_id)
    write_f5_f6(job_id)

    if len([f for f in job.files if f["ready"] == False]) == 0:
        job.files_ready = True
        job.date_completed = datetime.datetime.utcnow()
    else:
        job.files_failed = True

    try:
        if job.email_complete:
            if (not cfront_settings.get("debug_mode",
                                        False)) and cfront_settings.get(
                                            "mails_on_completion", False):
                mail.mail_completed_job(None, job)
    except mail.MailError as m:
        print "writing mail on complete failed for job {0}".format(job.id)
示例#6
0
def process_queue(ofs, stride):
    # start a transaction to compute spacers for any jobs which lack them
    # NOTE this is the second place in our code where spacers can be created
    # this should be fixed as we don't do the same checks here that we do in
    # the first -- at views_ajax/post_new.
    #
    #here we protect against the case where no spacers pass but don't do
    # any checking of the input query.

    possible_spacer_jobs = Session.query(Job).filter(
        Job.computed_spacers == False).all()
    selected_jobs = [
        j for j in possible_spacer_jobs
        if j.id % stride == ofs and j.genome_name != "rn5"
    ]

    for j in selected_jobs[:100]:
        with transaction.manager:
            try:
                print 'computing spacers {0}'.format(j.id)
                spacer_infos = webserver_db.compute_spacers(j.sequence)
                if (len(spacer_infos)) == 0:
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job=j, **spacer_info))
                j.computed_spacers = True
                Session.add(j)
            except JobERR, e:
                print "Excepted a job error during Spacer finding for Job: {0}".format(
                    j.id)
                print j.sequence
示例#7
0
def main(global_config, **settings):
    """ 
    This function returns a Pyramid WSGI application.
    """
    import inspect
    set_cfront_settings(settings)
    set_genomes_settings(settings)

    engine = engine_from_config(settings, 'sqlalchemy.')
    Session.configure(bind=engine)
    Base.metadata.bind = engine
    config = Configurator(settings=settings)

    #static routes
    config.add_static_view('/css', 'static/css', cache_max_age=3600)
    config.add_static_view('/js', 'static/js', cache_max_age=3600)
    config.add_static_view('/img', 'static/img', cache_max_age=3600)
    config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
    config.add_static_view('/files', 'static/files', cache_max_age=3600)

    #page routes
    config.add_route('about', '/about', factory=PageResourceFactory)
    config.add_route('batch', '/batch/{batch_key}', factory = BatchResourceFactory)
    config.add_route('submit', '/', factory=PageResourceFactory)
    config.add_route('job', '/job/{job_key}', factory=PageResourceFactory)
    config.add_route('readonly', '/readonly',factory=PageResourceFactory)
    config.add_route('readout', '/guides/{job_key}',factory=PageResourceFactory)
    config.add_route('nickase', '/nick/{job_key}',factory=PageResourceFactory)
    config.add_route('downloads', '/downloads/{job_key}',factory=PageResourceFactory)
    
    #export routes
    config.add_route('gb_all_nicks', '/export/nicks_gb/{job_key}',factory=JobResourceFactory)
    config.add_route('gb_one_nick', '/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',factory=JobResourceFactory)
    config.add_route('csv_one_spacer', '/export/spacer_csv/{job_key}/{spacerid}',factory=JobResourceFactory)
    config.add_route('csv_all_guides', '/export/csv_all_guides/{job_key}',factory=JobResourceFactory)
    config.add_route('gb_all_guides', '/export/guides_gb/{job_key}',factory=JobResourceFactory)
    

    #ajax routes
    config.add_route('job_check_spacers','/j/check_spacers/{job_key}',factory=JobResourceFactory)
    config.add_route('job_post_new', '/j/post_new')
    config.add_route('jobs_from_fasta', '/j/from_fasta')
    config.add_route('job_from_spacers', '/j/from_spacers')
    config.add_route('job_retrieve_spacers', '/j/retrieve_spacers/{job_key}',factory=JobResourceFactory)
    config.add_route('job_email_complete', '/j/email_complete/{job_key}',factory=JobResourceFactory)

    config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
    config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
    config.add_route('spacer_retrieve_regions', '/s/retrieve_regions/{spacer_id}')

    #rest routes
    config.add_route('job_rest', '/r/job/{job_key}',factory=JobResourceFactory)
    config.add_route('batch_rest', '/r/batch/{batch_key}',factory=BatchResourceFactory)
    config.add_route('spacer_rest', '/r/spacer/{spacer_id}')
    
    config.add_route('maintainance','/sorry')

    config.scan()
    return config.make_wsgi_app()
示例#8
0
 def estimated_waiting_time(self):        
     deltatime = timedelta(days = 7)
     t1 = datetime.now() - deltatime      
     possible_hit_jobs_total = Session.query(Job)\
                     .filter(Job.date_submitted > t1)\
                     .filter(Job.failed == False)\
                     .filter(Job.date_completed == None)\
                     .filter(Job.batchid == None)\
                     .order_by('-id')\
                     .limit(100)\
                     .all()
     ids = [i.id for i in possible_hit_jobs_total]
     if self.id not in ids:
       return 24*60*60
     else:
       m1 = ids.index(self.id)
       m2 = len(possible_hit_jobs_total)
       N = 12
       if (N - 1) <= m1:
         if m1 >= 50:
           p1 = 0.0
           p2 = 0.0
         else:
           p1 = 1.0/2.0*round(sum([scipy.misc.comb(m1,ii) for ii in range(0,N-1 + 1)]))/scipy.power(2,m1)
           p2 = 0.0
       else:
         if (m2-m1) >= 50:
           p1 = 0.5
           p2 = 0.0
         else:
           p1 = 0.5
           p2 = 1.0/2.0*round(sum([scipy.misc.comb(m2-m1,ii) for ii in range(0,N-1-m1 + 1)]))/scipy.power(2,m2-m1)
       p = p1 + p2
       print m1, m2, p1, p2, p
       return int(ceil((self.n_spacers - self.n_completed_spacers)/2.0) * ceil(1.0/p * 60.0))
示例#9
0
 def top_hits(self):
     from cfront.models import Hit
     return [
         e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
             Spacer.id == self.id).order_by(desc(Hit.score)).limit(
                 50).all()
     ]
示例#10
0
def write_f5_f6(job_id):
    job = Session.query(Job).get(job_id)
    f4p = job.f4
    f5p = job.f5
    f6p = job.f6
    gene = os.path.join(job.safe_name)

    if os.path.isfile(f5p):
        os.remove(f5p)
    if os.path.isfile(f6p):
        os.remove(f6p)

    if v: print "writing f5 at {0}".format(f5p)
    if v: print "writing f6 at {0}".format(f6p)

    cmd = ("R -f "+\
           "{0}/make_graphical_output.R --slave --vanilla --args "+\
           "{1} {2} {3} {4}")\
        .format(CDSCRIPTS,f4p,f5p,gene,f6p)

    if v: print "running cmd:"
    if v: print "  {0}".format(cmd)
    prc = spc.Popen(cmd, shell=True)
    out = prc.communicate()
    print "outval: {0}".format(out)

    if v: print "DONE"
    if v: print
示例#11
0
def write_f5_f6(job_id):
    job = Session.query(Job).get(job_id)
    f4p = job.f4
    f5p = job.f5
    f6p = job.f6
    gene= os.path.join(job.safe_name)
    
    if os.path.isfile(f5p):
        os.remove(f5p)    
    if os.path.isfile(f6p):
        os.remove(f6p)

    if v: print "writing f5 at {0}".format(f5p)
    if v: print "writing f6 at {0}".format(f6p)

    cmd = ("R -f "+\
           "{0}/make_graphical_output.R --slave --vanilla --args "+\
           "{1} {2} {3} {4}")\
        .format(CDSCRIPTS,f4p,f5p,gene,f6p)

    if v: print "running cmd:"
    if v: print "  {0}".format(cmd)
    prc = spc.Popen(cmd, shell=True)
    out = prc.communicate()
    print "outval: {0}".format(out)

    if v: print "DONE"
    if v: print
示例#12
0
def write_f4(job_id):
    job = Session.query(Job).get(job_id)
    f4p = job.f4
    if os.path.isfile(f4p):
        os.remove(f4p)
    
    if v: print "writing f4 at {0}".format(f4p)
    with open(f4p,"w") as f:
        f.write(" ".join(["SPACER","ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE", "MISMATCH", "MISMATCH_POS", "SCORE", "GENE"])+"\n")
        for spacer in job.good_spacers:
           ot_count=1
           for hit in sorted(spacer.hits, key=lambda x:-1* x.score):
               spacerid = "spacer_{0}".format(spacer.id)
               ontarget = 1 if hit.ontarget else 0
               if ontarget: 
                   hitid = "{0}".format(spacerid)
               else: 
                   hitid = "{0}_off_{1}".format(spacerid, ot_count) 
                   ot_count+=1
               chr=hit.chr[3:]
               pos=hit.start
               strand="+" if hit.strand == 1 else "-"
               sequence=hit.sequence
               mismatch=hit.n_mismatches
               
               mismatch_pos= "NA" if mismatch == 0 \
                             else ":".join( ["{0}".format(i) for i in range(20) 
                                             if spacer.sequence[i] != hit.sequence[i]])
               score = "{0:.1f}".format(hit.score)
               gene = hit.gene
               row = " ".join("{0}".format(e) for e in [spacerid, ontarget, hitid, chr, pos, strand, sequence, mismatch,
                                mismatch_pos, score, gene])
               f.write(row+"\n")
    if v: print "DONE"
    if v: print 
示例#13
0
def write_f4(job_id):
    job = Session.query(Job).get(job_id)
    f4p = job.f4
    if os.path.isfile(f4p):
        os.remove(f4p)

    if v: print "writing f4 at {0}".format(f4p)
    with open(f4p, "w") as f:
        f.write(" ".join([
            "SPACER", "ON_TARGET", "ID", "CHR", "POS", "STRAND", "SEQUENCE",
            "MISMATCH", "MISMATCH_POS", "SCORE", "GENE"
        ]) + "\n")
        for spacer in job.good_spacers:
            ot_count = 1
            for hit in sorted(spacer.hits, key=lambda x: -1 * x.score):
                spacerid = "spacer_{0}".format(spacer.id)
                ontarget = 1 if hit.ontarget else 0
                if ontarget:
                    hitid = "{0}".format(spacerid)
                else:
                    hitid = "{0}_off_{1}".format(spacerid, ot_count)
                    ot_count += 1
                chr = hit.chr[3:]
                pos = hit.start
                strand = "+" if hit.strand == 1 else "-"
                sequence = hit.sequence
                mismatch = hit.n_mismatches

                mismatch_pos= "NA" if mismatch == 0 \
                              else ":".join( ["{0}".format(i) for i in range(20)
                                              if spacer.sequence[i] != hit.sequence[i]])
                score = "{0:.1f}".format(hit.score)
                gene = hit.gene
                row = " ".join("{0}".format(e) for e in [
                    spacerid, ontarget, hitid, chr, pos, strand, sequence,
                    mismatch, mismatch_pos, score, gene
                ])
                f.write(row + "\n")
    if v: print "DONE"
    if v: print
示例#14
0
                    print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                    j.failed = True
                    Session.add(j)
                    print j.id
                    print j.sequence
                    raise e

    #change on 3/14/2016
    deltatime = timedelta(days = 7)
    t1 = datetime.now() - deltatime
    
    possible_hit_jobs = Session.query(Job)\
                    .filter(Job.date_submitted > t1)\
                    .filter(Job.failed == False)\
                    .filter(Job.date_completed == None)\
                    .filter(Job.batchid == None)\
                    .filter(Job.computed_spacers == True)\
                    .order_by('-id')\
                    .limit(100)\
                    .all()
    possible_hit_batch_jobs = Session.query(Job)\
                    .filter(Job.date_submitted > t1)\
                    .filter(Job.failed == False)\
                    .filter(Job.date_completed == None)\
                    .filter(Job.batchid != None)\
                    .filter(Job.computed_spacers == True)\
                    .order_by('-id')\
                    .limit(100)\
                    .all()
    selected_hit_jobs = possible_hit_jobs + possible_hit_batch_jobs
    
示例#15
0
                print "computing spacers {0}".format(j.id)
                spacer_infos = webserver_db.compute_spacers(j.sequence)
                if (len(spacer_infos)) == 0:
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job=j, **spacer_info))
                j.computed_spacers = True
                Session.add(j)
            except JobERR, e:
                print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                print j.sequence

    # starts a transaction to compute hits for any spacer which has not
    # "computed_hits" or "computing_hits"

    possible_hit_jobs = Session.query(Job).join(Spacer).filter(Spacer.score == None).filter(Job.failed == False).all()
    selected_hit_jobs = [j for j in possible_hit_jobs if j.id % stride == ofs and j.genome_name != "rn5"]

    if len(selected_hit_jobs) > 0:
        batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
        # sorts jobs to process recent submissions and non-batch jobs first
        def priority(j):
            # return 1 * j.id
            from random import random

            f = 1 if random() > 0.5 else -1
            if j.key == "7956546276189290":
                return -100000
            if j.batch is not None:
                return 100000000
            else:
示例#16
0
 def genic_hits(self):
     from cfront.models import Hit
     return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).filter(Hit.gene != None).all()]
示例#17
0
 def top_hits(self):
     from cfront.models import Hit
     return [e.toJSON() for e in Session.query(Hit).join(Spacer).filter(Spacer.id == self.id).order_by(desc(Hit.score)).limit(50).all()]
示例#18
0
           "{0}/make_graphical_output.R --slave --vanilla --args "+\
           "{1} {2} {3} {4}")\
        .format(CDSCRIPTS,f4p,f5p,gene,f6p)

    if v: print "running cmd:"
    if v: print "  {0}".format(cmd)
    prc = spc.Popen(cmd, shell=True)
    out = prc.communicate()
    print "outval: {0}".format(out)

    if v: print "DONE"
    if v: print


import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
    env = bootstrap(sys.argv[1])
    while 1:
        j = Session.query(Job)\
                   .filter(Job.files_failed == False)\
                   .filter(Job.files_ready == False)\
                   .join(Spacer)\
                   .group_by(Job.id)\
                   .having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
                   .order_by(Job.id.desc()).first()
        if j:
            with transaction.manager:
                commence_file_io(j.id)
        time.sleep(2)
示例#19
0
           "{0}/make_graphical_output.R --slave --vanilla --args "+\
           "{1} {2} {3} {4}")\
        .format(CDSCRIPTS,f4p,f5p,gene,f6p)

    if v: print "running cmd:"
    if v: print "  {0}".format(cmd)
    prc = spc.Popen(cmd, shell=True)
    out = prc.communicate()
    print "outval: {0}".format(out)

    if v: print "DONE"
    if v: print


import transaction
from pyramid.paster import bootstrap
if __name__ == "__main__":
    env = bootstrap(sys.argv[1])
    while 1:
        j = Session.query(Job)\
                   .filter(Job.files_failed == False)\
                   .filter(Job.files_ready == False)\
                   .join(Spacer)\
                   .group_by(Job.id)\
                   .having(func.sum(case([(Spacer.score == None,1)], else_=0)) == 0)\
                   .order_by(Job.id.desc()).first()
        if j:
            with transaction.manager:
                commence_file_io(j.id)
        time.sleep(2)
示例#20
0
 def get_batch_by_key(batch_key):
     b =  Session.query(Batch).filter(Batch.key == batch_key).first()
     if b is None:
         from cfront.models import BatchNOTFOUND
         raise BatchNOTFOUND("batch not found {0}".format(batch_key), None)
     return b
示例#21
0
文件: __init__.py 项目: bh0085/crispr
def main(global_config, **settings):
    """ 
    This function returns a Pyramid WSGI application.
    """
    import inspect
    set_cfront_settings(settings)
    set_genomes_settings(settings)

    engine = engine_from_config(settings, 'sqlalchemy.')
    Session.configure(bind=engine)
    Base.metadata.bind = engine
    config = Configurator(settings=settings)

    config.include('pyramid_mako')

    #static routes
    config.add_static_view('/v2/css', 'static/css', cache_max_age=3600)
    config.add_static_view('/v2/js', 'static/js', cache_max_age=3600)
    config.add_static_view('/img', 'static/img', cache_max_age=3600)
    config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
    config.add_static_view('/files', 'static/files', cache_max_age=3600)

    #page routes
    
    config.add_route('splash_v2', '/v2', factory=PageResourceFactory)
    config.add_route('gene_results_v2', '/v2/{assembly}/{geneid}/gene_results', factory=PageResourceFactory)
    config.add_route('submit_v2', '/v2/{assembly}/submit', factory=PageResourceFactory)
    config.add_route('genes_autocomplete_array', '/v2/{assembly}/gene_names.json')
    config.add_route('genes_autocomplete_info', '/v2/{assembly}/genes_info.json')
    config.add_route('gene_genbank', '/v2/{assembly}/{geneid}/gene.gb')
    config.add_route('gene_genbank_spacer', '/v2/{assembly}/{geneid}/{tool}/{guide_sequence}/guide.gb')
    config.add_route('gene_genbank_all_spacers', '/v2/{assembly}/{geneid}/top_spacers.gb')
    config.add_route('gene_genbank_selected_spacers', '/v2/{assembly}/{geneid}/selected_spacers.gb')
    config.add_route('gene_fasta', '/v2/{assembly}/{geneid}/gene.fa')
    config.add_route('gene_genbank_json', '/v2/{assembly}/{geneid}/genbank.json')
    config.add_route('gene_sequence', '/v2/{assembly}/{geneid}/sequence.json')
    config.add_route('job_data','/v2/{assembly}/{geneid}/job.json')
    config.add_route('email_all_genbank', '/v2/{assembly}/{geneid}/{email}/email_all_genbank')

    
    config.add_route('about', '/about', factory=PageResourceFactory)
    config.add_route('batch', '/v1/batch/{batch_key}', factory = BatchResourceFactory)
    config.add_route('submit', '/', factory=PageResourceFactory)
    config.add_route('job', '/v1/job/{job_key}', factory=PageResourceFactory)
    config.add_route('readonly', '/readonly',factory=PageResourceFactory)
    config.add_route('readout', '/v1/guides/{job_key}',factory=PageResourceFactory)
    config.add_route('nickase', '/v1/nick/{job_key}',factory=PageResourceFactory)
    config.add_route('downloads', '/v1/downloads/{job_key}',factory=PageResourceFactory)

    
    #export routes
    config.add_route('gb_all_nicks', '/v1/export/nicks_gb/{job_key}',factory=JobResourceFactory)
    config.add_route('gb_one_nick', '/v1/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',factory=JobResourceFactory)
    config.add_route('csv_one_spacer', '/v1/export/spacer_csv/{job_key}/{spacerid}',factory=JobResourceFactory)
    config.add_route('csv_all_guides', '/v1/export/csv_all_guides/{job_key}',factory=JobResourceFactory)
    config.add_route('gb_all_guides', '/v1/export/guides_gb/{job_key}',factory=JobResourceFactory)
    
    #v2 ajax routes
    config.add_route('v2_query_gene', '/v2/{assembly}/{geneid}/{email}/query_gene')

    #ajax routes
    config.add_route('job_check_spacers','/j/check_spacers/{job_key}',factory=JobResourceFactory)
    config.add_route('job_post_new', '/j/post_new')
    config.add_route('jobs_from_fasta', '/j/from_fasta')
    config.add_route('job_from_spacers', '/j/from_spacers')
    config.add_route('job_retrieve_spacers', '/j/retrieve_spacers/{job_key}',factory=JobResourceFactory)
    config.add_route('job_email_complete', '/j/email_complete/{job_key}',factory=JobResourceFactory)

    config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
    config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
    config.add_route('spacer_retrieve_regions', '/s/retrieve_regions/{spacer_id}')

    #rest routes
    config.add_route('job_rest', '/r/job/{job_key}',factory=JobResourceFactory)
    config.add_route('batch_rest', '/r/batch/{batch_key}',factory=BatchResourceFactory)
    config.add_route('spacer_rest', '/r/spacer/{spacer_id}')
    
    config.add_route('maintainance','/sorry')

    config.scan()
    return config.make_wsgi_app()
示例#22
0
def main(global_config, **settings):
    """ 
    This function returns a Pyramid WSGI application.
    """
    import inspect
    set_cfront_settings(settings)
    set_genomes_settings(settings)

    engine = engine_from_config(settings, 'sqlalchemy.')
    Session.configure(bind=engine)
    Base.metadata.bind = engine
    config = Configurator(settings=settings)

    #static routes
    config.add_static_view('/css', 'static/css', cache_max_age=3600)
    config.add_static_view('/js', 'static/js', cache_max_age=3600)
    config.add_static_view('/img', 'static/img', cache_max_age=3600)
    config.add_static_view('/pages', 'static/pages', cache_max_age=3600)
    config.add_static_view('/files', 'static/files', cache_max_age=3600)

    #page routes
    config.add_route('about', '/about', factory=PageResourceFactory)
    config.add_route('batch',
                     '/batch/{batch_key}',
                     factory=BatchResourceFactory)
    config.add_route('submit', '/', factory=PageResourceFactory)
    config.add_route('job', '/job/{job_key}', factory=PageResourceFactory)
    config.add_route('readonly', '/readonly', factory=PageResourceFactory)
    config.add_route('readout',
                     '/guides/{job_key}',
                     factory=PageResourceFactory)
    config.add_route('nickase', '/nick/{job_key}', factory=PageResourceFactory)
    config.add_route('downloads',
                     '/downloads/{job_key}',
                     factory=PageResourceFactory)

    #export routes
    config.add_route('gb_all_nicks',
                     '/export/nicks_gb/{job_key}',
                     factory=JobResourceFactory)
    config.add_route('gb_one_nick',
                     '/export/nicks_gb/{job_key}/{spacerfwdid}/{spacerrevid}',
                     factory=JobResourceFactory)
    config.add_route('csv_one_spacer',
                     '/export/spacer_csv/{job_key}/{spacerid}',
                     factory=JobResourceFactory)
    config.add_route('csv_all_guides',
                     '/export/csv_all_guides/{job_key}',
                     factory=JobResourceFactory)
    config.add_route('gb_all_guides',
                     '/export/guides_gb/{job_key}',
                     factory=JobResourceFactory)

    #ajax routes
    config.add_route('job_check_spacers',
                     '/j/check_spacers/{job_key}',
                     factory=JobResourceFactory)
    config.add_route('job_post_new', '/j/post_new')
    config.add_route('jobs_from_fasta', '/j/from_fasta')
    config.add_route('job_from_spacers', '/j/from_spacers')
    config.add_route('job_retrieve_spacers',
                     '/j/retrieve_spacers/{job_key}',
                     factory=JobResourceFactory)
    config.add_route('job_email_complete',
                     '/j/email_complete/{job_key}',
                     factory=JobResourceFactory)

    config.add_route('spacer_retrieve_hits', '/s/retrieve_hits/{spacer_id}')
    config.add_route('spacer_check_hits', '/s/check_hits/{spacer_id}')
    config.add_route('spacer_retrieve_regions',
                     '/s/retrieve_regions/{spacer_id}')

    #rest routes
    config.add_route('job_rest',
                     '/r/job/{job_key}',
                     factory=JobResourceFactory)
    config.add_route('batch_rest',
                     '/r/batch/{batch_key}',
                     factory=BatchResourceFactory)
    config.add_route('spacer_rest', '/r/spacer/{spacer_id}')

    config.add_route('maintainance', '/sorry')

    config.scan()
    return config.make_wsgi_app()
示例#23
0
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job=j, **spacer_info))
                j.computed_spacers = True
                Session.add(j)
            except JobERR, e:
                print "Excepted a job error during Spacer finding for Job: {0}".format(
                    j.id)
                print j.sequence

    #starts a transaction to compute hits for any spacer which has not
    # "computed_hits" or "computing_hits"

    possible_hit_jobs =  Session.query(Job)\
                    .join(Spacer)\
                    .filter(Spacer.score == None)\
                    .filter(Job.failed == False)\
                     .all()
    selected_hit_jobs = [
        j for j in possible_hit_jobs
        if j.id % stride == ofs and j.genome_name != "rn5"
    ]

    if len(selected_hit_jobs) > 0:
        batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]

        #sorts jobs to process recent submissions and non-batch jobs first
        def priority(j):
            #return 1 * j.id
            from random import random
            f = 1 if random() > .5 else -1
示例#24
0
 def get_job_by_key(job_key):
     j = Session.query(Job).filter(Job.key == job_key).first()
     if j is None:
         from cfront.models import JobNOTFOUND
         raise JobNOTFOUND("job not found {0}".format(job_key), None)
     return j
示例#25
0
                print j.id
            except JobERR, e:
                if e.message == Job.NOSPACERS:
                    print "No spacers in JOB ID: {0}".format(j.id)
                elif e.message == Job.ERR_INVALID_CHARACTERS:
                    print e.message
                else:
                    print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                    print j.id
                    print j.sequence
                    raise e


    possible_hit_jobs =  Session.query(Job)\
                    .join(Spacer)\
                    .filter(Spacer.score == None)\
                    .filter(Job.failed == False)\
                     .all()
    selected_hit_jobs = possible_hit_jobs


    procs = []
    max_procs = 8
    manager = Manager()
    jobs_q = JoinableQueue()


    r = redis.Redis()

    dcount = 0 
    while(1):
示例#26
0
 def get_job_by_key(job_key):
     j =  Session.query(Job).filter(Job.key == job_key).first()
     if j is None:
         from cfront.models import JobNOTFOUND
         raise JobNOTFOUND("job not found {0}".format(job_key), None)
     return j
示例#27
0
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("inifile",
                        help="inifile to configure the server")
    args = parser.parse_args()
    
    #starts up the paster env
    env = bootstrap(args.inifile)
    jobpath = cfront_settings["jobs_directory"]


    #for d in os.listdir(jobpath):
    #        print "removing directory: {0}".format(d)
    #        shutil.rmtree(os.path.join(jobpath,d))
    #        print os.path.join(jobpath,d)

    with transaction.manager:
            for j in Session.query(Job).order_by(Job.id).all():
                if j.genome_name == "hg19" :
                    continue
                else:
                    for s in j.spacers:
                        Session.delete(s)
                    print "deleting spacers for job {0}".format(j.id)
                    j.computed_spacers = False
                    j.files_ready=False
                    j.date_completed = None
                    j.files_failed =False
                    j.failed = False
示例#28
0
 def genic_hits(self):
     from cfront.models import Hit
     return [
         e.toJSON() for e in Session.query(Hit).join(Spacer).filter(
             Spacer.id == self.id).filter(Hit.gene != None).all()
     ]
示例#29
0
def process_queue(ofs, stride):
    deltatime = timedelta(days = 7)
    t1 = datetime.now() - deltatime
    possible_spacer_jobs = Session.query(Job)\
                                  .filter(Job.date_submitted > t1)\
                                  .filter(Job.failed == False)\
                                  .filter(Job.computed_spacers == False)\
                                  .all()
    selected_jobs = possible_spacer_jobs
    
    for j in selected_jobs[:100]:
        with transaction.manager:
            try:
                print 'computing spacers {0}'.format(j.id)
                spacer_infos = webserver_db.compute_spacers(j.sequence)  
                if(len(spacer_infos)) == 0:
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job = j,**spacer_info))
                j.computed_spacers = True
                Session.add(j)
                print j.id
            except JobERR, e:
                if e.message == Job.NOSPACERS:
                    j.failed = True
                    j.computed_spacers = True
                    Session.add(j)
                    print "No spacers in JOB ID: {0}".format(j.id)
                elif e.message == Job.ERR_INVALID_CHARACTERS:
                    j.failed = True
                    j.computed_spacers = True
                    Session.add(j)
                    print e.message
                else:
                    print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                    j.failed = True
                    Session.add(j)
                    print j.id
                    print j.sequence
                    raise e
示例#30
0
                    raise JobERR(Job.NOSPACERS, j)
                for spacer_info in spacer_infos:
                    Session.add(Spacer(job = j,**spacer_info))
                j.computed_spacers = True
                Session.add(j)
            except JobERR, e:
                print "Excepted a job error during Spacer finding for Job: {0}".format(j.id)
                print j.sequence

    
    #starts a transaction to compute hits for any spacer which has not
    # "computed_hits" or "computing_hits"

    possible_hit_jobs =  Session.query(Job)\
                    .join(Spacer)\
                    .filter(Spacer.score == None)\
                    .filter(Job.failed == False)\
                     .all()
    selected_hit_jobs = [j for j in possible_hit_jobs if j.id % stride == ofs and j.genome_name != "rn5"]


    if len(selected_hit_jobs) > 0:
        batched_jobs = [j for j in selected_hit_jobs if j.batch is not None]
        #sorts jobs to process recent submissions and non-batch jobs first
        def priority(j):
            #return 1 * j.id
            from random import random
            f = 1 if random() > .5 else -1
            if j.key == "7956546276189290" : return -100000
            if j.batch is not None: return 100000000
            else: return f* j.id 
示例#31
0
import transaction
import os, shutil

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("inifile", help="inifile to configure the server")
    args = parser.parse_args()

    #starts up the paster env
    env = bootstrap(args.inifile)
    jobpath = cfront_settings["jobs_directory"]

    #for d in os.listdir(jobpath):
    #        print "removing directory: {0}".format(d)
    #        shutil.rmtree(os.path.join(jobpath,d))
    #        print os.path.join(jobpath,d)

    with transaction.manager:
        for j in Session.query(Job).order_by(Job.id).all():
            if j.genome_name == "hg19":
                continue
            else:
                for s in j.spacers:
                    Session.delete(s)
                print "deleting spacers for job {0}".format(j.id)
                j.computed_spacers = False
                j.files_ready = False
                j.date_completed = None
                j.files_failed = False
                j.failed = False