def delete_tasks(): period = request.args(0) if not period in ['1d', '3d', '1w', '1m', '3m']: return '' now = s.utc_time and request.utcnow or request.now if period == '1d': limit = timed(days=1) elif period == '3d': limit = timed(days=3) elif period == '1w': limit = timed(days=7) elif period == '1m': limit = timed(days=30) elif period == '3m': limit = timed(days=90) limit = now - limit if request.vars.confirm == '1': statuses = ['COMPLETED', 'FAILED', 'EXPIRED', 'STOPPED'] dbs( (st.status.belongs(statuses)) & (st.start_time < limit) ).delete() sc_cache.clear("plugin_cs_monitor.*") redirect(URL('index')) limit = limit.strftime('%Y-%m-%d %H:%M:%S') return dict(limit=limit)
def setup_task(__OPENGNSYS_SERVER__, __APIKEY__, ou_id, lab_id, pc_id, ip, maxtime): ## habra que cambiar timed(seconds=30) por timed(hours=maxtime) #current.scheduler.queue_task('send_poweroff',[ip, __APIKEY__], # start_time=datetime.now() + timed(minutes=__TMP_MINUTOS_RESERVA__)) current.scheduler.queue_task( 'unreserve', [__OPENGNSYS_SERVER__, __APIKEY__, ou_id, lab_id, pc_id], start_time=datetime.now() + timed(hours=maxtime))
def generate_stats(self, message): group_id = message["peer_id"] - 2000000000 current_time = int(time.time()) time_for = current_time - 24*60*60 # get start of today dt = datetime.combine(date.today(), timed(0, 0, 0)) # start of yesterday = one day before start of today sday_timestamp = int((dt - timedelta(days=1)).timestamp()) # end of yesterday = one second before start of today eday_timestamp = int((dt - timedelta(seconds=1)).timestamp()) yesterday_messages = db.messages.find({'group_id':group_id, 'date':{'$gt':sday_timestamp, '$lt':eday_timestamp}}).count() all_messages = db.messages.find({'group_id':group_id, 'date':{'$gt':time_for}}).sort([("from_id", pymongo.DESCENDING)]) result = "💬За последние 24 часа: {}\n За вчерашний день: {}\Написали мне тут:\n".format(all_messages.count(), yesterday_messages) segregate_users = all_messages.limit(10).distinct("from_id") top_users = [] for user in segregate_users: msgs_from_user = db.messages.find({'group_id':group_id, 'date':{'$gt':time_for}, 'from_id':user}) text_list = [] msgs_from_user_text = db.messages.find({'group_id':group_id, 'date':{'$gt':time_for}, 'from_id':user}) for msgs in msgs_from_user_text: if msgs["text"]: splitted_msg = msgs["text"].split() y = [s for s in splitted_msg if len(s) > 3] text_list.extend(y) counter = collections.Counter(text_list) top_users.append((user, msgs_from_user.count(), msgs_from_user, counter.most_common())) sorted_top_users = sorted(top_users, key=lambda tup: tup[1], reverse=True) for i in sorted_top_users[:10]: #print(i[3]) try: result += "{} {}.: {} сбщ. ({} - {} раз, {} - {} раз, {} - {} раз)\n".format(i[2][0]["first_name"], i[2][0]["last_name"][0], i[1], i[3][0][0][:100], i[3][0][1], i[3][1][0][:100], i[3][1][1], i[3][2][0][:100], i[3][2][1]) except IndexError: pass last_hour = current_time - 60*60 top1hour_messages = db.messages.find({'group_id':group_id, 'date':{'$gt':last_hour}}).sort([("from_id", pymongo.DESCENDING)]) if top1hour_messages: top_users_segregation = top1hour_messages.limit(5).distinct("from_id") top_users_1hr = [] for user in segregate_users: msgs_from_user = db.messages.find({'group_id':group_id, 'date':{'$gt':last_hour}, 'from_id':user}) top_users_1hr.append((user, msgs_from_user.count(), msgs_from_user)) sorted_1hr = sorted(top_users_1hr, key=lambda tup: tup[1], reverse=True) try: result += "За последний час больше всего написал: {} {} ({} сбщ)".format(sorted_1hr[0][2][0]["first_name"], sorted_1hr[0][2][0]["last_name"], sorted_1hr[0][1]) except Exception: pass return result
def queryNonWorkshopSessions(self, request): """Query for all non-workshop sessions before 7 pm.""" # fetch all sessions whose start time is before 7 pm sessions = Session.query( ndb.AND(Session.startTime != None, Session.startTime <= timed(hour=19)) ) # return individual SessionForm object per Session return SessionForms( items=[self._copySessionToForm( ses, getattr(ses.key.parent().get(), 'name')) for ses in sessions if ses.typeOfSession != "workshop"])
def workers(): now = s.utc_time and request.utcnow or request.now limit = now - timed(seconds=s.heartbeat * 3 * 10) w = dbs(sw.id > 0).select(orderby=~sw.id) for row in w: if row.last_heartbeat < limit: row.status_ = nice_worker_status('Probably Dead') else: row.status_ = nice_worker_status(row.status) BASEURL = URL("plugin_cs_monitor", "wactions", user_signature=True) return dict(w=w, BASEURL=BASEURL, limit=limit)
def getNonWorkshopsBeforeSevenPm(self, request): """Get all sessions that are not workshops happening before 7 pm""" sessions = Session.query(ndb.AND(Session.startTime != None, Session.startTime <= timed(hour=19))) filtered_sessions = [] for session in sessions: # for each session check that that session type is not a # 'workshop' or 'Workshop' if not 'workshop' in session.typeOfSession and not 'Workshop' in session.typeOfSession: filtered_sessions.append(session) return SessionForms( items=[self._copySessionToForm(session) for session in filtered_sessions] )
def workers(): session.forget(response) now = s.utc_time and request.utcnow or request.now limit = now - timed(seconds=s.heartbeat * 3 * 10) w = s.get_workers() for worker, props in w.iteritems(): if props.last_heartbeat < limit: props.status_ = nice_worker_status('Probably Dead') else: props.status_ = nice_worker_status(props.status) props.worker_stats_ = nice_worker_stats(props.worker_stats) BASEURL = URL("plugin_cs_monitor", "wactions", user_signature=True) return dict(w=w, BASEURL=BASEURL, limit=limit)
def getEarlyNonWorkshopSessions(self, request): """Returns non-workshop sessions occurring before 7pm""" sessions = Session.query( ndb.AND(Session.startTime != None, Session.startTime <= timed(hour=19))) filtered_sessions = [] for session in sessions: if 'workshop' in session.typeOfSession: continue else: filtered_sessions.append(session) return SessionForms(items=[ self._copySessionToForm(session) for session in filtered_sessions ])
def getSessionsByTypeTime(self, request): """Returns all non workshop sessions held before 7 pm. """ sessions = Session.query( ndb.AND(Session.startTime != None, Session.startTime <= timed(hour=19))) filtered_sessions = [] for session in sessions: if 'workshop' == session.typeOfSession: continue else: filtered_sessions.append(session) return SessionForms(items=[ self._copySessionToForm(session) for session in filtered_sessions ])
def getNonWorkshopBefore7pm(self, request): """Returns all non-workshop sessions occurring before 7pm""" # query the sessions that starts before 7pm sessions = Session.query(ndb.AND( Session.startTime != None, Session.startTime <= timed(hour=19))) # filter for non-workshop sessions filtered_sessions = [] for session in sessions: if session.typeOfSession != 'workshop': filtered_sessions.append(session) return SessionForms( items=[self._copySessionToForm(session) for session in filtered_sessions] )
def getEarlyNonWorkshopSessions(self, request): """Returns non-workshop sessions occurring before 7pm""" sessions = Session.query(ndb.AND( Session.startTime != None, Session.startTime <= timed(hour=19) )) filtered_sessions = [] for session in sessions: if 'workshop' in session.typeOfSession: continue else: filtered_sessions.append(session) return SessionForms( items=[self._copySessionToForm(session) for session in filtered_sessions] )
def getBefore7pmNonWorkshopSessions(self, request): """Returns non-workshop sessions that take place before 7pm""" sessions = Session.query(ndb.AND( Session.startTime != None, Session.startTime <= timed(hour=19) )) nonworkshop_sessions = [] for session in sessions: if 'workshop' in session.typeOfSession: continue else: nonworkshop_sessions.append(session) return SessionForms( items=[self._copySessionToForm(session) for session in nonworkshop_sessions] )
def getBefore7PMWorkshopSessions(self, request): """Returns sessions that occur before 7PM and are not a workshop""" # First filter sessions based on time using NBD sessions = Session.query(ndb.AND(Session.startTime != None, Session.startTime <= timed(hour=19))) # Second, fileter session type using native Python since NBD cannot # handle two inequality filters in one query time_filter_sessions = [] for session in sessions: if "workshop" in session.typeOfSession: continue else: time_filter_sessions.append(session) return SessionForms(items=[self._copySessionToForm(session) for session in time_filter_sessions])
def analyze_task(): task_id = request.args(0) if not task_id: return '' task = dbs(st.id == task_id).select().first() if not task: return '' q = sr.task_id == task_id first_run = dbs(q).select(sr.start_time, orderby=sr.start_time, limitby=(0,1)).first() last_run = dbs(q).select(sr.start_time, orderby=~sr.start_time, limitby=(0,1)).first() if not first_run: mode = 'no_runs' #we can rely on the data on the scheduler_task table because no scheduler_run were found q = st.id == task_id else: mode = 'runs' if len(request.args) >= 2: if request.args(1) == 'byfunction': if mode == 'runs': q = sr.task_id.belongs(dbs(st.function_name == task.function_name)._select(st.id)) else: q = st.function_name == task.function_name elif request.args(1) == 'bytaskname': if mode == 'runs': q = sr.task_id.belongs(dbs(st.task_name == task.task_name)._select(st.id)) else: q = st.task_name == task.task_name elif request.args(1) == 'this': if mode == 'runs': q = sr.task_id == task_id else: q = st.id == task_id if len(request.args) == 4 and request.args(2) == 'byday': daysback = int(request.args(3)) now = s.utc_time and request.utcnow or request.now day = now.date() - timed(days=daysback) if mode == 'runs': q = q & ((sr.start_time >= day) & (sr.start_time < day + timed(days=1))) else: q = q & ((st.last_run_time >= day) & (st.last_run_time < day + timed(days=1))) if mode == 'runs': gb_duration_rows, jgb_duration_series = gb_duration(q) jgb_duration_series = dumps(jgb_duration_series) else: #no duration can be calculated using the scheduler_task table only jgb_duration_series = dumps([]) gb_status_rows, jgb_status_series = gb_status(q, mode) jgb_status_series = dumps(jgb_status_series) gb_when_rows, jgb_when_series = bydate(q, mode) jgb_when_series = dumps(jgb_when_series) if len(request.args) == 4 and request.args(2) == 'byday': gb_whend_rows, jgb_whend_series = byday(q, day, mode) jgb_whend_series = dumps(jgb_whend_series) else: jgb_whend_series = dumps([[]]) return locals()
def run_vidjil(id_file, id_config, id_data, grep_reads, clean_before=False, clean_after=False): from subprocess import Popen, PIPE, STDOUT, os from datetime import timedelta as timed if db.sequence_file[id_file].pre_process_flag == "FAILED" : print("Pre-process has failed") raise ValueError('pre-process has failed') return "FAIL" ## re schedule if pre_process is still pending if db.sequence_file[id_file].pre_process_flag not in ["COMPLETED", "DONE"] and db.sequence_file[id_file].pre_process_flag: print("Pre-process is still pending, re-schedule") args = [id_file, id_config, id_data, grep_reads] task = scheduler.queue_task("vidjil", args, repeats = 1, timeout = defs.TASK_TIMEOUT, start_time=request.now + timed(seconds=1200)) db.results_file[id_data] = dict(scheduler_task_id = task.id) db.commit() print(task.id) sys.stdout.flush() return "SUCCESS" ## les chemins d'acces a vidjil / aux fichiers de sequences upload_folder = defs.DIR_SEQUENCES out_folder = defs.DIR_OUT_VIDJIL_ID % id_data cmd = "rm -rf "+out_folder p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() ## filepath du fichier de séquence row = db(db.sequence_file.id==id_file).select() filename = row[0].data_file output_filename = defs.BASENAME_OUT_VIDJIL_ID % id_data seq_file = upload_folder+filename ## config de vidjil vidjil_cmd = db.config[id_config].command if 'next' in vidjil_cmd: vidjil_cmd = vidjil_cmd.replace('next', '') vidjil_cmd = vidjil_cmd.replace(' germline' , defs.DIR_GERMLINE_NEXT) cmd = defs.DIR_VIDJIL_NEXT + '/vidjil-algo ' else: vidjil_cmd = vidjil_cmd.replace(' germline' , defs.DIR_GERMLINE) cmd = defs.DIR_VIDJIL + '/vidjil-algo ' if grep_reads: # TODO: security, assert grep_reads XXXX vidjil_cmd += ' --grep-reads "%s" ' % grep_reads os.makedirs(out_folder) out_log = out_folder+'/'+output_filename+'.vidjil.log' vidjil_log_file = open(out_log, 'w') try: ## commande complete cmd += ' -o ' + out_folder + " -b " + output_filename cmd += ' ' + vidjil_cmd + ' '+ seq_file ## execute la commande vidjil print("=== Launching Vidjil ===") print(cmd) print("========================") sys.stdout.flush() p = Popen(cmd, shell=True, stdin=PIPE, stdout=vidjil_log_file, stderr=STDOUT, close_fds=True) (stdoutdata, stderrdata) = p.communicate() print("Output log in " + out_log) sys.stdout.flush() db.commit() ## Get result file if grep_reads: out_results = out_folder + '/seq/clone.fa-1' else: out_results = out_folder + '/' + output_filename + '.vidjil' print("===>", out_results) results_filepath = os.path.abspath(out_results) stream = open(results_filepath, 'rb') except: print("!!! Vidjil failed, no result file") res = {"message": "[%s] c%s: Vidjil FAILED - %s" % (id_data, id_config, out_folder)} log.error(res) raise ## Parse some info in .log vidjil_log_file.close() segmented = re.compile("==> segmented (\d+) reads \((\d*\.\d+|\d+)%\)") windows = re.compile("==> found (\d+) .*-windows in .* segments .* inside (\d+) sequences") info = '' reads = None segs = None ratio = None wins = None for l in open(out_log): m = segmented.search(l) if m: print(l, end=' ') segs = int(m.group(1)) ratio = m.group(2) info = "%d segmented (%s%%)" % (segs, ratio) continue m = windows.search(l) if m: print(l, end=' ') wins = int(m.group(1)) reads = int(m.group(2)) info = "%d reads, " % reads + info + ", %d windows" % wins break ## insertion dans la base de donnée ts = time.time() db.results_file[id_data] = dict(status = "ready", run_date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S'), data_file = stream ) db.commit() if clean_after: clean_cmd = "rm -rf " + out_folder p = Popen(clean_cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() ## l'output de Vidjil est stocké comme resultat pour l'ordonnanceur ## TODO parse result success/fail config_name = db.config[id_config].name res = {"message": "[%s] c%s: Vidjil finished - %s - %s" % (id_data, id_config, info, out_folder)} log.info(res) if not grep_reads: for row in db(db.sample_set_membership.sequence_file_id==id_file).select() : sample_set_id = row.sample_set_id print(row.sample_set_id) compute_extra(id_file, id_config, 5) run_fuse(id_file, id_config, id_data, sample_set_id, clean_before = False) return "SUCCESS"
def run_vidjil(id_file, id_config, id_data, grep_reads, clean_before=False, clean_after=False): from subprocess import Popen, PIPE, STDOUT, os from datetime import timedelta as timed ## re schedule if pre_process is still pending if db.sequence_file[id_file].pre_process_flag == "WAIT" or db.sequence_file[ id_file].pre_process_flag == "RUN": print "Pre-process is still pending, re-schedule" args = [id_file, id_config, id_data, grep_reads] task = scheduler.queue_task("vidjil", args, repeats=1, timeout=defs.TASK_TIMEOUT, start_time=request.now + timed(seconds=1200)) db.results_file[id_data] = dict(scheduler_task_id=task.id) db.commit() print task.id sys.stdout.flush() return "SUCCESS" if db.sequence_file[id_file].pre_process_flag == "FAILED": print "Pre-process has failed" raise ValueError('pre-process has failed') return "FAIL" ## les chemins d'acces a vidjil / aux fichiers de sequences germline_folder = defs.DIR_GERMLINE upload_folder = defs.DIR_SEQUENCES out_folder = defs.DIR_OUT_VIDJIL_ID % id_data cmd = "rm -rf " + out_folder p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() ## filepath du fichier de séquence row = db(db.sequence_file.id == id_file).select() filename = row[0].data_file output_filename = defs.BASENAME_OUT_VIDJIL_ID % id_data seq_file = upload_folder + filename ## config de vidjil vidjil_cmd = db.config[id_config].command vidjil_cmd = vidjil_cmd.replace(' germline', germline_folder) if grep_reads: # TODO: security, assert grep_reads XXXX vidjil_cmd += ' -FaW "%s" ' % grep_reads os.makedirs(out_folder) out_log = out_folder + '/' + output_filename + '.vidjil.log' vidjil_log_file = open(out_log, 'w') try: ## commande complete cmd = defs.DIR_VIDJIL + '/vidjil ' + ' -o ' + out_folder + " -b " + output_filename cmd += ' ' + vidjil_cmd + ' ' + seq_file ## execute la commande vidjil print "=== Launching Vidjil ===" print cmd print "========================" sys.stdout.flush() p = Popen(cmd, shell=True, stdin=PIPE, stdout=vidjil_log_file, stderr=STDOUT, close_fds=True) (stdoutdata, stderrdata) = p.communicate() print "Output log in " + out_log sys.stdout.flush() db.commit() ## Get result file if grep_reads: out_results = out_folder + '/seq/clone.fa-1' else: out_results = out_folder + '/' + output_filename + '.vidjil' print "===>", out_results results_filepath = os.path.abspath(out_results) stream = open(results_filepath, 'rb') except: print "!!! Vidjil failed, no result file" res = { "message": "[%s] c%s: Vidjil FAILED - %s" % (id_data, id_config, out_folder) } log.error(res) raise ## Parse some info in .log vidjil_log_file.close() segmented = re.compile("==> segmented (\d+) reads \((\d*\.\d+|\d+)%\)") windows = re.compile( "==> found (\d+) .*-windows in .* segments .* inside (\d+) sequences") info = '' reads = None segs = None ratio = None wins = None for l in open(out_log): m = segmented.search(l) if m: print l, segs = int(m.group(1)) ratio = m.group(2) info = "%d segmented (%s%%)" % (segs, ratio) continue m = windows.search(l) if m: print l, wins = int(m.group(1)) reads = int(m.group(2)) info = "%d reads, " % reads + info + ", %d windows" % wins break ## insertion dans la base de donnée ts = time.time() db.results_file[id_data] = dict(status="ready", run_date=datetime.datetime.fromtimestamp( ts).strftime('%Y-%m-%d %H:%M:%S'), data_file=stream) db.commit() if clean_after: clean_cmd = "rm -rf " + out_folder p = Popen(clean_cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() ## l'output de Vidjil est stocké comme resultat pour l'ordonnanceur ## TODO parse result success/fail config_name = db.config[id_config].name res = { "message": "[%s] c%s: Vidjil finished - %s - %s" % (id_data, id_config, info, out_folder) } log.info(res) if not grep_reads: for row in db( db.sample_set_membership.sequence_file_id == id_file).select(): sample_set_id = row.sample_set_id print row.sample_set_id run_fuse(id_file, id_config, id_data, sample_set_id, clean_before=False) return "SUCCESS"