def selected(auth_info, **kwargs): if auth_info['code'] == 1: return json.dumps(auth_info) username = auth_info['username'] try: data = request.get_json()['params'] where = data.get('where', None) m_table = data.get('m_table', None) field = data.get('field', None) s_table = data.get('s_table', None) res = app.config['db'].get_one_result(m_table, [field], where) res = res[field].split(',') #eg: ['1', '2'] result = app.config['db'].get_results(s_table, ['id', 'name', 'name_cn']) for x in result: #eg: [{'id':1, 'name':'sa'},{'id': 2, 'name': 'php'}] for r_id in res: if int(r_id) == int(x['id']): x['selected'] = 'selected="selected"' utils.write_log('api').info('%s selected %s successfully' % (username, s_table)) return json.dumps({'code': 0, 'result': result}) except: utils.write_log('api').error('selected error %s' % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': 'selected.get faild'})
def apply_create(auth_info, **kwargs): username = auth_info['username'] r_id = auth_info['r_id'] field = [ 'project_id', 'info', 'applicant', 'commit', 'apply_date', 'status', 'detail' ] try: data = request.get_json()[ 'params'] # project_id,project_name,applicant,info,detail data['commit'] = '11111' # 脚本获取最新的commit data['apply_date'] = time.strftime('%Y-%m-%d %H:%M') data['status'] = 1 # 项目申请后状态变为1, data['applicant'] = username where = {"project_id": int(data['project_id'])} data.pop('project_username') # 因为申请表单里面有项目名,数据库里存的上id,所以要除掉 res = app.config['db'].get_one_result('project_apply', field, where) if res and res['status'] in (1, 2): return json.dumps({'code': 1, 'errmsg': '目前项目状态不可申请'}) if not res: app.config['db'].execute_insert_sql('project_apply', data) else: app.config['db'].execute_update_sql('project_apply', data, where) app.config['db'].execute_insert_sql('project_deploy', data) utils.write_log('api').info('%s:项目申请成功' % username) return json.dumps({'code': 0, 'result': '项目申请成功'}) except: utils.write_log('api').error('project apply error: %s' % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': '项目申请失败'})
def userselfinfo(**kwargs): fields = ['id', 'username', 'name', 'email', 'mobile', 'is_lock', 'r_id'] try: user = app.config['cursor'].get_one_result( 'user', fields, where={'username': username}) if user.get('r_id', None): r_id = user['r_id'].split(',') rids = app.config['cursor'].get_results('role', ['id', 'name', 'p_id'], where={'id': r_id}) else: rids = {} pids = [] for x in rids: pids += x['p_id'].split(',') pids = list(set(pids)) #去重。通过用户名查到其角色id,在通过角色id取到用户的权限id user['r_id'] = [x['name'] for x in rids] #将角色id转为角色名 if pids: #将用户的权限id转为权限名 mypids = app.config['cursor'].get_results( 'power', ['id', 'name', 'name_cn', 'url'], where={'id': pids}) user['p_id'] = dict([ (str(x['name']), dict([(k, x[k]) for k in ('name_cn', 'url')])) for x in mypids ]) #返回格式:{'git':{'name_cn':'git','url':'http://git.com'},......} else: user['p_id'] = {} utils.write_log('api').info(username, 'get_user_info') return json.dumps({'code': 0, 'user': user}) except: utils.write_log('api').error("Get users list error: %s" % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': 'get userinfo failed'})
def test(model, test_loader, criterion, to_log=None): model.eval() test_loss = 0 correct = 0 output_array = 0 target_array = 0 with torch.no_grad(): for data, target in test_loader: # target = target.long() data, target = data.to(device), target.to(device) output = model(data) o_array = output.cpu().numpy() t_array = target.cpu().numpy() test_loss += criterion(output, target) # sum up batch loss mse = rmse(output, target) # pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability # correct += pred.eq(target.view_as(pred)).sum().item() test_loss /= len(test_loader.sampler) test_loss *= test_loader.batch_size # acc = 100. * correct / len(test_loader.sampler) acc = 0 format_str = 'Test set: Average loss: {:.4f}, MSE: ({:3.5f})\n'.format(test_loss, mse) print(format_str) if to_log is not None: write_log(format_str, to_log) return test_loss.item(), acc
def update_log(args): anime_name, episodes = read_args(args) episodes = utils.compress_range(episodes) utils.write_log(anime_name, episodes) utils.update_tracklist(anime_name, episodes) utils.write_cache(anime_name) utils.update_watchlater(anime_name, episodes)
def __init__(self, filter_id, group_id, req, out_dir=".", order = None, min_identity = 30, min_blast_evalue = 0.01, save_to_ITOL = False, sep = ",", report_unfit = False): self.req = req self.order = order self.filter_id = filter_id self.group_id = group_id self.out_dir = os.path.abspath(out_dir) self.min_identity = min_identity self.min_blast_evalue = min_blast_evalue self.save_to_ITOL = save_to_ITOL self.sep = sep self.report_unfit = report_unfit if self.order == None: ## user didn't override the order argument d = os.path.abspath(os.path.dirname(__file__)) data_env = os.path.join(d, 'data/') self.order = utils.get_order(self.req,data_env) ## get blast versions self.blastp = self._get_blast_version("blastp") self.makeblastdb = self._get_blast_version("makeblastdb") utils.assure_path_exists(os.path.join(self.out_dir,group_id+ "_GROUP")) utils.write_log(os.path.join(self.out_dir,group_id + "_GROUP", "LOG"), "STEP 4 : GROUP ORFs", vars(self), "")
def test(model, test_loader, criterion, to_log=None): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in test_loader: # target = target.long() data, target = data.to(device), target.to(device) output, mu, logvar = model(data) loss, bce, kld = loss_fn(output, target, mu, logvar) test_loss += loss # test_loss += criterion(output, target) # sum up batch loss # mse = rmse(output, target) # pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability # correct += pred.eq(target.view_as(pred)).sum().item() t = torch.tensor([0.5]).to(device) # threshold out = (output > t).float() * 1 pre = precision(target, out) rec = recall(target, out) # if pre > 0.7 and rec> 0.7 and data.size()[1] > 20: # tar_npy = target.cpu().numpy() # out_npy = out.cpu().numpy() # plt.imshow(tar_npy) # plt.imshow(out_npy) test_loss /= len(test_loader.sampler) test_loss *= test_loader.batch_size # acc = 100. * correct / len(test_loader.sampler) format_str = 'Test set: Average loss: {:.4f}, precision: {:.2%}, recall: {:.2%}\n'.format( test_loss, pre, rec) print(format_str) if to_log is not None: write_log(format_str, to_log) return test_loss.item(), pre, rec
def _delete_sql(self, table_name, where): if not (where and isinstance(where, dict)): return "" where_cond = ["%s='%s'" % (k, v) for k,v in where.items()] sql = "DELETE FROM %s WHERE %s" % (table_name, ' AND '.join(where_cond)) utils.write_log('api').info("Delete sql: %s" % sql) return sql
def validation(val_dataLoader, criterion, args): print("******* begin validation *******") model.eval() all_loss = 0.0 EVAL = Eval(args.n_classes) EVAL.reset() with torch.no_grad(): with tqdm(total=len(val_dataLoader), unit='batch') as pbar: for batch_id, (data, target_mask) in enumerate(val_dataLoader): data, target_mask = data.to(device), target_mask.to(device) out = model(data) loss = criterion(out, target_mask) current_loss = loss.data.item() all_loss += current_loss out = out.data.cpu().numpy() target_mask = target_mask.data.cpu().numpy() EVAL.add_batch(target_mask, out.argmax(axis=1)) pbar.update(1) print('[ validation ] [average loss:{}]'.format(all_loss/len(val_dataLoader))) PA = EVAL.Pixel_Accuracy() MPA = EVAL.Mean_Pixel_Accuracy() MIoU = EVAL.Mean_Intersection_over_Union() FWIoU = EVAL.Frequency_Weighted_Intersection_over_Union() print('[ validation ] [PA1: {:.8f}], [MPA1: {:.8f}], [MIoU1: {:.8f}], [FWIoU1: {:.8f}]'.format(PA, MPA,MIoU, FWIoU)) log_message ='**validation** [average loss: {:.8f} ],[PA1: {:.8f}], [MPA1: {:.8f}], [MIoU1: {:.8f}], [FWIoU1: {:.8f}]'\ .format(all_loss/len(val_dataLoader),PA, MPA,MIoU, FWIoU) write_log(args.log_file_name,log_message) return FWIoU
def train(epoch, train_dataLoader, optimizer, criterion, args, best_pred): print("******* begin train *******") model.train() all_loss = 0.0 with tqdm(total= len(train_dataLoader), unit='batch') as pbar: for batch_id, (data, target_img) in enumerate(train_dataLoader): #global_step+=1 # lr = lr_function(global_step, WARMUP_STEP, TOTAL_STEP) # adjust_lr(optimizer,lr) scheduler(optimizer, batch_id, epoch, best_pred) data,target_img = data.to(device), target_img.to(device) optimizer.zero_grad() out = model(data) loss = criterion(out,target_img) current_loss = loss.item() all_loss += current_loss avg_loss = all_loss/(batch_id+1) loss.backward() optimizer.step() pbar.update(1) postfix_message={"cur_loss":"%.6f"%(current_loss) , "avg_loss ":"%.6f"%(avg_loss) , "lr":"%.8f"%(optimizer.param_groups[0]['lr'])} pbar.set_postfix(log = postfix_message) write_log(args.log_file_name,"[ epoch {} ]".format(epoch)) log_message ="**train** [ avg_loss {:.8f} ] , [ lr {:.8f} ]".format(avg_loss,optimizer.param_groups[0]['lr']) write_log(args.log_file_name,log_message) return avg_loss
def test(model, test_loader, criterion, to_log=None): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for (data, target) in test_loader: target = target.long() data = torch.permute(data, (0, 2, 1, 3, 4)) data, target = data.to(device), target.to(device) output = model(data) loss = criterion(output, target) test_loss += loss pred = output.argmax( dim=1, keepdim=True) # get the index of the max log-probability correct += pred.eq(target.view_as(pred)).sum().item() test_loss /= len(test_loader.sampler) test_loss *= test_loader.batch_size acc = 100. * correct / len(test_loader.sampler) format_str = 'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format( test_loss, correct, len(test_loader.sampler), acc) print(format_str) if to_log is not None: write_log(format_str, to_log) return test_loss.item(), acc
def gitolite(): git_confile = app.config['git_confile'] api_dir = os.path.dirname(os.path.realpath(__file__)) script_dir = os.path.join(api_dir.rstrip('api'), 'script') projects, pro_pri = utils.project_members() group = utils.role_members() try: # 将项目和用户信息写入配置文件中 with open(git_confile, 'w') as f: str0 = "" for k, v in group.items(): str0 += "@%s = %s\n" % (k, " ".join(v)) f.write(str0) f.write("\n") str1 = "" for k, v in projects.items(): # 由于projests中存放了项目所有的成员,包括负责人。需要把负责人剔除 v = list(set(v) - set(pro_pri[k])) str1 += "repo %s \n" % k str1 += " RW+ = %s \n" % ' '.join(pro_pri[k]) # 负责人的权限最大 if v: # 如果项目除了负责人外,有其他成员,则设置成员的权限 str1 += " - master = %s \n" % (' '.join(v) ) # 项目成员无权操作master分支 str1 += " RW = %s \n" % (' '.join(v)) # 项目成员可以操作其他分支 f.write(str1) # git add/commit/push生效.路径暂时写死,定版前修改 # stdout=utils.run_script_with_timeout("sh %s/git.sh" % script_dir) # print stdout return {'code': 0, 'result': "git操作成功"} except: utils.write_log('api').error("get config error: %s" % traceback.format_exc()) return {'code': 1, 'errmsg': "get config error"}
def execute_insert_sql(self, table_name, data): sql = self._insert_sql(table_name, data) if sql: return self._execute(sql) else: utils.write_log('api').error('execute_insert_sql missing sql') return None
def __init__(self, flags): torch.set_default_tensor_type('torch.cuda.FloatTensor') # fix the random seed or not fix_seed() self.setup_path(flags) self.network = mlp.MLPNet(num_classes=flags.num_classes) self.network = self.network.cuda() print(self.network) print('flags:', flags) if not os.path.exists(flags.logs): os.mkdir(flags.logs) flags_log = os.path.join(flags.logs, 'flags_log.txt') write_log(flags, flags_log) self.load_state_dict(flags.state_dict) self.configure(flags)
def test(model, test_loader, criterion, to_log=None): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for (azi, ele, target) in test_loader: # prepare input and target to device azi = azi.to(device, dtype=torch.float) ele = ele.to(device, dtype=torch.float) target = target.to(device, dtype=torch.long) output = model(azi, ele) loss = criterion(output, target) test_loss += loss pred = output.argmax( dim=1, keepdim=True) # get the index of the max log-probability correct += pred.eq(target.view_as(pred)).sum().item() test_loss /= len(test_loader.sampler) test_loss *= test_loader.batch_size acc = 100. * correct / len(test_loader.sampler) format_str = 'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format( test_loss, correct, len(test_loader.sampler), acc) print(format_str) if to_log is not None: write_log(format_str, to_log) return test_loss.item(), acc
def create_job_list(results_dir, prep_dir, scan_dir, args): # keeping a text file of all the genomes used log_other = "### INPUT ### \ngenome\thmmer_result\n" jobs = [] for f in os.listdir(scan_dir): if f.endswith(".result"): basename = os.path.basename(f) basename = basename.replace(".result", "") scan_summary = copy.copy(args) scan_summary["basename"] = basename scan_summary["out_file"] = os.path.join(results_dir, basename + ".csv") scan_summary["results_file"] = os.path.join( scan_dir, basename + ".result") scan_summary["orf_locs"] = os.path.join(prep_dir, basename + ".bed") scan_summary["orfs"] = {} scan_summary["unfit_orfs"] = {} if scan_summary["report_unfit"]: scan_summary["report_unfit"] = os.path.join( scan_summary["report_unfit"], basename + ".csv") jobs.append(scan_summary) log_other = log_other + basename + "\t" + f + "\n" utils.write_log(os.path.join(results_dir, "LOG"), "STEP 3 : FILTER HMMER HITS", args, log_other) return jobs
def role_select(auth_info,**kwargs): username = auth_info['username'] if '1' not in auth_info['r_id']: return json.dumps({'code': 1,'errmsg':'you not admin,no power' }) try: output = ['id','name','name_cn','p_id','info'] data = request.get_json()['params'] fields = data.get('output', output) #查询权限表,生产id2name的字典 result = app.config['db'].get_results('power', ['id', 'name']) power_name = dict([(str(x['id']), x['name']) for x in result]) #将角色对应的p_id都转为name,最终返回的结果p_id的值都是name result = [] res = app.config['db'].get_results('role', fields) for x in res: p_name = [power_name[p_id] for p_id in x['p_id'].split(',') if p_id in power_name] x['p_id'] = ','.join(p_name) #将原有x['p_id']中的id转为name result.append(x) utils.write_log('api').info('%s select role list success' % username) return json.dumps({'code':0,'result':result,'count':len(result)}) except: utils.write_log('api').error("select role list error: %s" % traceback.format_exc()) return json.dumps({'code':1,'errmsg':'get rolelist failed'})
def create_jobs_list(args, prep_dir, scan_dir): ''' create a list of scanning jobs for HMMsearch while generating a LOG file in the SCAN directory''' # get version of hmmscan for log file log_other = get_hmmer_version(args.hmmsearch) log_other = log_other + get_hmmer_version(args.hmmpress) # keeping a text file of all the genomes used log_other = log_other + "### INPUT ### \ncnt\tgenome\tprep_fasta_file\n" jobs = [] for f in os.listdir(prep_dir): if f.endswith(".fasta"): basename = os.path.basename(f) basename = basename.replace(".fasta", "") prep_file = os.path.join(prep_dir, f) scan_genome = { "basename": basename, "fasta_file": prep_file, "out_dir": scan_dir, "hmm_db": args.hmm_db, "hmmsearch": args.hmmsearch } jobs.append(scan_genome) log_other = log_other + \ str(len(jobs)) + "\t" + basename + "\t" + \ prep_file + "\n" utils.write_log(os.path.join(scan_dir, "LOG"), "STEP 2 : GENOME SCANNING", vars(args), log_other) return jobs
def execute_update_sql(self, table_name, data, where, fields=None): sql = self._update_sql(table_name, data, where, fields) if sql: return self._execute(sql) else: utils.write_log('api').error('execute_update_sql missing sql') return None
def server_getlist(auth_info, **kwargs): username = auth_info['username'] if '1' not in auth_info['r_id']: return json.dumps({'code': 1, 'errmsg': 'you have no power'}) try: output = [ 'server.id', 'server.host_name', 'server.name_cn', 'server.ip', 'server.account', 'server.admin_username', 'server.admin_password', 'server.sg_id', 'server.comment', 'server.outerip', 'server_group.name_cn' ] data = request.get_json()['params'] fields = data.get('output', output) where = data.get('where', {}) where['server.sg_id'] = 'server_group.id' join = data.get('join', True) results = app.config['db'].get_results('server,server_group', fields, where, join=join) print results if not results: return json.dumps({'code': 1, 'errmsg': 'data not exist'}) utils.write_log('api').info('%s select server list success' % username) return json.dumps({ 'code': 0, 'result': results, 'count': len(results) }) except: utils.write_log('error').error('select server list faild %s' % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': 'select role list fail'})
def train(model, data_loader, criterion, optimizer, epoch=0, to_log=None, print_freq=25): # create Average Meters batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() train_loss = [] # switch to train mode model.train() # record start time start = time.time() for i, (azi, ele, target) in enumerate(data_loader): # prepare input and target to device azi = azi.to(device, dtype=torch.float) ele = ele.to(device, dtype=torch.float) target = target.to(device, dtype=torch.long) # measure data loading time data_time.update(time.time() - start) # zero the parameter gradients optimizer.zero_grad() # gradient and do SGD step output = model(azi, ele) loss = criterion(output, target) train_loss.append(loss.item()) loss.backward() optimizer.step() # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), data_loader.batch_size) top1.update(prec1.item(), data_loader.batch_size) top5.update(prec5.item(), data_loader.batch_size) # measure elapsed time batch_time.update(time.time() - start) start = time.time() # print training info if i % print_freq == 0: str = ('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:3.3f} ({top1.avg:3.3f})\t' 'Prec@5 {top5.val:3.3f} ({top5.avg:3.3f})'.format( epoch, i, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5)) print(str) if to_log is not None: write_log(str + '\n', to_log) return train_loss
def role_getone(auth_info, **kwargs): username = auth_info['username'] try: output = ['id', 'name', 'name_cn', 'p_id', 'info'] data = request.get_json()['params'] fields = data.get('output', output) where = data.get('where', None) result = app.config['db'].get_one_result('role', fields, where) if not result: return json.dumps({'code': 1, 'errmsg': 'data not exist'}) #查询power表生成id,name字典 power_results = app.config['db'].get_results('power', ['id', 'name']) power_dict = dict((str(x['id']), x['name']) for x in power_results) p_name = [ power_dict[p_id] for p_id in result['p_id'].split(',') if p_id in power_dict ] result['p_id'] = ','.join(p_name) utils.write_log('api').info('%s select role success' % username) return json.dumps({'code': 0, 'result': result}) except: utils.write_log('api').error('select role by name or id faild %s' % traceback.format_exc()) return json.dumps({ 'code': 1, 'errmsg': 'select role by name or id faild' })
def login(): try: username = request.args.get('username', None) passwd = request.args.get('passwd', None) passwd = hashlib.md5(passwd).hexdigest() if not (username and passwd): return json.dumps({'code': 1, 'errmsg': "need username or passwd"}) result = app.config['db'].get_one_result( 'user', ['id', 'username', 'password', 'r_id', 'is_lock'], {'username': username}) if not result: return json.dumps({'code': 1, 'errmsg': "user is not exist"}) if result['is_lock'] == 1: return json.dumps({'code': 1, 'errmsg': "user is lock"}) if passwd == result['password']: data = {'last_login': time.strftime('%Y-%m-%d %H:%M:%S')} app.config['db'].execute_update_sql('user', data, {'username': username}) token = utils.get_validate(result['username'], result['id'], result['r_id'], app.config['passport_key']) utils.write_log('api').info("%s login sucess" % username) return json.dumps({'code': 0, 'authorization': token}) else: return json.dumps({'code': 1, 'errmsg': "passwd is wrong"}) except: utils.write_log('api').error("login error: %s" % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': "login fail"})
def login(): try: username = request.args.get('username', None) passwd = request.args.get('passwd', None) passwd = hashlib.md5(passwd).hexdigest() if not (username and passwd): return josin.dumps({'code': 1, 'errmsg': '需要输入用户名和密码'}) result = app.config['db'].get_one_result( 'user', ['id', 'username', 'password', 'r_id', 'is_lock'], {'username': username}) if not result: return json.dumps({"code": 1, "errmsg": "用户不存在"}) if result['is_lock'] == 1: return json.dumps({"code": 1, "errmsg": "用户已锁定"}) if passwd == result['password']: data = {'last_login': time.strftime('%Y-%m-%d %H:%M:%S')} app.config['db'].execute_update_sql('user', data, {'username': username}) token = utils.get_validate(result['username'], result['id'], result['r_id'], app.config['passport_key']) utils.write_log('api').info('%s login success' % username) return json.dumps({'code': 0, 'authorization': token}) else: return json.dumps({'code': 1, "errmsg": "输入密码有误"}) except: utils.write_log('api').error('login error: %s', traceback.format_exc()) return json.dumps({"code": 1, "errmsg": "login fail"})
def role_getlist(auth_info, **kwargs): username = auth_info['username'] if '1' not in auth_info['r_id']: return json.dumps({'code': 1, 'errmsg': 'you have no power'}) try: output = ['id', 'name', 'name_cn', 'p_id', 'info'] data = request.get_json()['params'] fields = data.get('output', output) where = data.get('where', None) results = app.config['db'].get_results('role', fields, where) if not results: return json.dumps({'code': 1, 'errmsg': 'data not exist'}) power_results = app.config['db'].get_results('power', ['id', 'name']) power_dict = dict((str(x['id']), x['name']) for x in power_results) for result in results: p_name = [ power_dict[p_id] for p_id in result['p_id'].split(',') if p_id in power_dict ] result['p_id'] = ','.join(p_name) utils.write_log('api').info('%s select role list success' % username) return json.dumps({ 'code': 0, 'result': results, 'count': len(results) }) except: utils.write_log('error').errot('select role list faild %s' % traceback.format_exc()) return json.dumps({'code': 1, 'errmsg': 'select role list faild'})
def run(self): ## get domains' lengths and domains to ignore self._parse_domains() self._parse_domains_to_ignore() log_other = "### INPUT ### \ngenome\tfasta_hmmer_result\tgff_hmmer_result\n" # keeping a text file of all the genomes used jobs = [] for file in os.listdir(os.path.join(self.args["out_dir"],self.args["scan_id"] + "_SCAN")): if file.endswith(".sixframe.result"): basename = os.path.basename(file) basename = basename.replace(".sixframe.result","") scan_summary = dict(self.args) scan_summary["basename"] = basename scan_summary["out_file"] = os.path.join(self.args["results_dir"], basename + ".csv") scan_summary["orf_locs"] = None scan_summary["orfs"] = {} scan_summary["unfit_orfs"] = {} if scan_summary["report_unfit"]: scan_summary["report_unfit"] = os.path.join(self.args["results_dir"],"UNFIT",basename + ".csv") jobs.append(scan_summary) for j in jobs: sixframe_file, annotation_file = run_summarise(j) log_other = log_other + j["basename"] + "\t" + sixframe_file + "\t" + annotation_file + "\n" utils.write_log(os.path.join(self.args["results_dir"], "LOG"), "STEP 3 : FILTER HMMER HITS", self.args, log_other)
def _select_sql(self, table_name, fields, where=None, order=None, asc_order=True, limit=None, join=False): if isinstance(where, dict) and where: conditions = [] for k, v in where.items(): if join and '.' in k and '.' in v: conditions.append("%s=%s" % (k, v)) elif isinstance(v, list): conditions.append('%s IN (%s)' % (k, ','.join(v))) elif isinstance(v, str) or isinstance(v, unicode): conditions.append("%s='%s'" % (k, v)) elif isinstance(v, int): conditions.append('%s=%s' % (k, v)) sql = 'SELECT %s FROM %s WHERE %s' % (','.join(fields), table_name, ' AND '.join(conditions)) elif not where: sql = 'SELECT %s FROM %s' % (','.join(fields), table_name) else: sql = '' if order and (isinstance(order, str) or isinstance(order, unicode)): sql = '%s ORDER BY %s %s' % (sql, order, 'ASC' if asc_order else 'DESC') if limit and isinstance(limit, tuple) and len(limit) == 2: sql = '%s LIMIT %s,%s' % (sql, limit[0], limit[1]) utils.write_log('api').info('Select sql: %s' % sql) return sql
def server_getone(auth_info, **kwargs): username = auth_info['username'] if '1' not in auth_info['r_id']: return json.dumps({'code': 1, 'errmsg': 'you have no power'}) try: output = [ 'id', 'host_name', 'name_cn', 'ip', 'account', 'admin_username', 'admin_password', 'sg_id', 'comment', 'outerip' ] data = request.get_json()['params'] fields = data.get('output', output) where = data.get('where', None) result = app.config['db'].get_one_result('server', fields, where) if not result: return json.dumps({'code': 1, 'errmsg': 'data not exist'}) sg_results = app.config['db'].get_results('server_group', ['id', 'name_cn']) sg_dict = dict((str(x['id']), x['name_cn']) for x in sg_results) print sg_dict print result sg_name = sg_dict.get(result['sg_id']) result['sg_id'] = sg_name utils.write_log('api').info('%s select server success' % username) return json.dumps({'code': 0, 'result': result}) except: utils.write_log('api').error('select server by name or id faild %s' % traceback.format_exc()) return json.dumps({ 'code': 1, 'errmsg': 'select server by name or id faild' })
def get_results(self, table_name, fields, where=None, order=None, asc_order=True, limit=None, join=False): sql = self._select_sql(table_name, fields, where, order, asc_order, limit, join) if sql: self._execute(sql) results_all_set = self._fetchall() if results_all_set: results_list = [] for result in results_all_set: results_list.append( dict([(item, '' if result[index] is None else result[index]) for index, item in enumerate(fields)])) return results_list else: return {} utils.write_log('api').error('get_results missing sql') return None
def on_epoch_end(self, epoch, logs=None): y_pred = self.model.predict(self.x_valid).flatten() y_true = self.y_valid.flatten() auc = roc_auc_score(y_true=y_true, y_score=y_pred) # roc曲线的auc precision, recall, _thresholds = precision_recall_curve( y_true=y_true, probas_pred=y_pred) aupr = m.auc(recall, precision) y_pred = [1 if prob >= self.threshold else 0 for prob in y_pred] acc = accuracy_score(y_true=y_true, y_pred=y_pred) f1 = f1_score(y_true=y_true, y_pred=y_pred) print(type(aupr)) logs['val_aupr'] = float(aupr) logs['val_auc'] = float(auc) logs['val_acc'] = float(acc) logs['val_f1'] = float(f1) logs['dataset'] = self.dataset logs['aggregator_type'] = self.aggregator_type logs['kfold'] = self.k logs['epoch_count'] = epoch + 1 print( f'Logging Info - epoch: {epoch+1}, val_auc: {auc}, val_aupr: {aupr}, val_acc: {acc}, val_f1: {f1}' ) write_log('log/train_history.txt', logs, mode='a')
def train(self): self.train_setup() self.sess.run(tf.global_variables_initializer()) # Load the pre-trained model if provided if self.conf.pretrain_file is not None: self.load(self.loader, self.conf.pretrain_file) # Start queue threads. threads = tf.train.start_queue_runners(coord=self.coord, sess=self.sess) # Train! for step in range(self.conf.num_steps+1): start_time = time.time() feed_dict = { self.curr_step : step } loss_value = 0 # Clear the accumulated gradients. self.sess.run(self.zero_op, feed_dict=feed_dict) # Accumulate gradients. for i in range(self.conf.grad_update_every): _, l_val = self.sess.run([self.accum_grads_op, self.reduced_loss], feed_dict=feed_dict) loss_value += l_val # Normalise the loss. loss_value /= self.conf.grad_update_every # Apply gradients. if step % self.conf.save_interval == 0: images, labels, summary, _ = self.sess.run( [self.image_batch, self.label_batch, self.total_summary, self.train_op], feed_dict=feed_dict) self.summary_writer.add_summary(summary, step) self.save(self.saver, step) else: self.sess.run(self.train_op, feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration)) write_log('{:d}, {:.3f}'.format(step, loss_value), self.conf.logfile) # finish self.coord.request_stop() self.coord.join(threads)
def train(self): self.train_setup() self.sess.run(tf.global_variables_initializer()) # Load the pre-trained model if provided if self.conf.pretrain_file is not None: self.load(self.loader, self.conf.pretrain_file) # Start queue threads. threads = tf.train.start_queue_runners(coord=self.coord, sess=self.sess) # Train! for step in range(self.conf.num_steps+1): start_time = time.time() feed_dict = { self.curr_step : step } if step % self.conf.save_interval == 0: loss_value, images, labels, preds, summary, _ = self.sess.run( [self.reduced_loss, self.image_batch, self.label_batch, self.pred, self.total_summary, self.train_op], feed_dict=feed_dict) self.summary_writer.add_summary(summary, step) self.save(self.saver, step) else: loss_value, _ = self.sess.run([self.reduced_loss, self.train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration)) write_log('{:d}, {:.3f}'.format(step, loss_value), self.conf.logfile) # finish self.coord.request_stop() self.coord.join(threads)
def __parse_prompt(self, prompt, our_team): """ Determine piece to be moved and move required from prompt. """ # setup return defaults piece, up, right, hold_move, user_feedback = None, None, None, True, None # first check for special commands if prompt.lower()[:5] == 'debug': level_text = prompt[5:].strip() user_feedback = set_debugging_level(level_text, feedback_required=True) return piece, up, right, hold_move, user_feedback elif prompt.lower() == 'redraw': print(self.board.draw_board()) return piece, up, right, hold_move, user_feedback elif prompt.lower()[:4] == 'list': piece_ref_text = prompt[4:].strip() piece = self.get_piece(piece_ref_text) if piece: self.get_all_possible_moves(list_moves=True, pieces={piece.ref: piece}) else: self.get_all_possible_moves(list_moves=True) return piece, up, right, hold_move, user_feedback elif prompt.lower() == 'log': if self.logging: write_log(LOG) user_feedback = "Log written to current working directory" else: user_feedback = "Logging not currently enabled, change in literals.py" return piece, up, right, hold_move, user_feedback # attempt to get details of piece to be moved (first two chars as current cell_ref) try: current_cell_ref = prompt[:2] [cur_row, cur_col_no] = cell_ref_to_pos(current_cell_ref) except (IndexError, ValueError): user_feedback = ('A valid cell for your current position could not be found\n' + '(using the first two characters from your entry: "{0}")').format(prompt) return piece, up, right, hold_move, user_feedback piece_ref = self.board.get_piece_ref(cur_row, cur_col_no) if piece_ref: piece = self.get_piece(piece_ref) try: # todo consider more efficient ways to achieve the same assert ([cur_row, cur_col_no] in [our_team[obj].pos for obj in our_team]) except AssertionError: user_feedback = ('A piece in your team could not be found in cell: {0}\n' + '(using the first two characters from your entry)').format(current_cell_ref) return piece, up, right, hold_move, user_feedback # use last two characters as new cell_ref try: [new_row, new_col_no] = cell_ref_to_pos(prompt[-2:]) except ValueError: user_feedback = ( 'A valid new cell could not be identified from your input: {0}'.format(prompt)) return piece, up, right, hold_move, user_feedback up, right = new_row - cur_row, new_col_no - cur_col_no debug('piece_ref: {0} | up: {1} | right: {2}'.format(piece.ref, up, right), DebugLevel.mid) # attempt to get destination... try: assert up in range(-8, 9) and right in range(-8, 9) except AssertionError: user_feedback = ( 'A valid new cell could not be identified from your input: {0}'.format(prompt)) return piece, up, right, hold_move, user_feedback hold_move = False return piece, up, right, hold_move, user_feedback
def message_handler(messages, bus_interface, upgrade_page, uninstall_page, install_page, home_page, inhibit_obj): for message in messages: try: (signal_type, action_content) = message if signal_type == "ready-download-start": (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_ready(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_ready(pkg_name) elif signal_type == 'ready-download-finish': (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_wait(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_wait(pkg_name) elif signal_type == 'pkgs-not-in-cache': (not_in_cache, action_type) = action_content utils.write_log("pkgs-not-in-cache:%s, action_type:%s" % (not_in_cache, action_type)) if action_type == ACTION_INSTALL: pass elif action_type == ACTION_UPGRADE: upgrade_page.upgrading_view.show_error("pkgs_not_in_cache", json.loads(not_in_cache)) elif signal_type == 'pkgs-mark-failed': (pkg_list, action_type) = action_content utils.write_log("pkgs-mark-failed:%s, action_type:%s" % (pkg_list, action_type)) if action_type == ACTION_INSTALL: pass elif action_type == ACTION_UPGRADE: upgrade_page.upgrading_view.show_error("pkgs_mark_failed", pkg_list) elif signal_type == 'marked-delete-system-pkgs': (pkgs, action_type) = action_content utils.write_log("marked-delete-system-pkgs:%s, action_type:%s" % (pkgs, action_type)) if action_type == ACTION_INSTALL: pass elif action_type == ACTION_UPGRADE: upgrade_page.upgrading_view.show_error("marked_delete_system_pkgs", json.loads(pkgs)) elif signal_type == 'pkgs-parse-download-error': (error, action_type) = action_content utils.write_log("pkgs-parse-download-error:%s, action_type:%s" % (error, action_type)) if action_type == ACTION_INSTALL: pass elif action_type == ACTION_UPGRADE: upgrade_page.upgrading_view.show_error("pkgs_parse_download_error", error) elif signal_type == "download-start": (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_start(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_start(pkg_name) elif signal_type == "download-update": (pkg_name, action_type, data) = action_content (percent, speed, finish_number, total, downloaded_size, total_size) = data if action_type == ACTION_INSTALL: install_page.download_update(pkg_name, percent, speed) elif action_type == ACTION_UPGRADE: upgrade_page.download_update(pkg_name, percent, speed, finish_number, total, downloaded_size, total_size) elif signal_type == "download-finish": (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_finish(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_finish(pkg_name) elif signal_type == "download-stop": (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_stop(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_stop(pkg_name) elif signal_type == "download-failed": (pkg_name, action_type, error) = action_content utils.write_log("download-failed:%s, action_type:%s" % (error, action_type)) if action_type == ACTION_INSTALL: install_page.download_failed(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_failed(pkg_name, error) elif signal_type == "action-start": (pkg_name, action_type) = action_content if action_type == ACTION_UNINSTALL: uninstall_page.action_start(pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.action_start(pkg_name) elif action_type == ACTION_INSTALL: install_page.action_start(pkg_name) inhibit_obj.set_inhibit() elif signal_type == "action-update": (pkg_name, action_type, percent, status) = action_content if action_type == ACTION_UNINSTALL: uninstall_page.action_update(pkg_name, percent) elif action_type == ACTION_UPGRADE: upgrade_page.action_update(pkg_name, percent, utils.l18n_status_info(status)) elif action_type == ACTION_INSTALL: install_page.action_update(pkg_name, percent) elif signal_type == "action-finish": (pkg_name, action_type, pkg_info_list) = action_content if action_type == ACTION_UNINSTALL: uninstall_page.action_finish(pkg_name, pkg_info_list) utils.show_notify(_("Uninstall %s Successfully.") % pkg_name, _("Uninstall")) elif action_type == ACTION_UPGRADE: upgrade_page.action_finish(pkg_name, pkg_info_list) global_event.emit("upgrade-finish-action", pkg_info_list) utils.set_last_upgrade_time() upgrade_page.refresh_status(pkg_info_list) utils.show_notify(_("%s packages upgrade successfully.") % len(pkg_info_list), _("Upgrade")) elif action_type == ACTION_INSTALL: install_page.action_finish(pkg_name, pkg_info_list) utils.show_notify(_("Install %s Successfully.") % pkg_name, _("Install")) inhibit_obj.unset_inhibit() refresh_current_page_status(pkg_name, pkg_info_list, bus_interface) if action_type != ACTION_UPGRADE: bus_interface.request_status( reply_handler=lambda reply: request_status_reply_hander( reply, install_page, upgrade_page, uninstall_page, pkg_info_list), error_handler=lambda e: action_finish_handle_dbus_error(pkg_info_list), ) elif signal_type == 'action-failed': # FIXME: change failed action dealing (pkg_name, action_type, pkg_info_list, errormsg) = action_content pkg_list = [str(info[0]) for info in pkg_info_list] utils.write_log("action-failed:%s, action_type:%s, pkg_list: %s" % (errormsg, action_type, pkg_list)) if action_type == ACTION_UNINSTALL: uninstall_page.action_finish(pkg_name, pkg_info_list) elif action_type == ACTION_UPGRADE: upgrade_page.upgrading_view.show_error("upgrade_failed", errormsg) utils.set_last_upgrade_time() elif action_type == ACTION_INSTALL: install_page.action_finish(pkg_name, pkg_info_list) inhibit_obj.unset_inhibit() refresh_current_page_status(pkg_name, pkg_info_list, bus_interface) bus_interface.request_status( reply_handler=lambda reply: request_status_reply_hander( reply, install_page, upgrade_page, uninstall_page), error_handler=lambda e:handle_dbus_error("request_status", e), ) elif signal_type == "update-list-update": percent = "[%i%%] " % float(action_content[0]) status = str(action_content[1]) speed_str = str(action_content[2]) status = utils.update_l18n_status_info(status) list_message = [] list_message.append(percent + status) list_message.append(speed_str) global_event.emit("show-message", list_message) elif signal_type == "update-list-merge": global_event.emit("show-message", [_("Generating package list database..."), ''], 0) elif signal_type == "update-list-finish": upgrade_page.fetch_upgrade_info() bus_interface.request_status( reply_handler=lambda reply: request_status_reply_hander(reply, install_page, upgrade_page, uninstall_page), error_handler=lambda e:handle_dbus_error("request_status", e), ) global_event.emit("show-message", [_("Package lists refreshed completely."), ''], 5000) global_event.emit('update-list-finish') global_event.emit("hide-update-list-dialog") elif signal_type == 'update-list-failed': # FIXME: change failed action dealing upgrade_page.fetch_upgrade_info() bus_interface.request_status( reply_handler=lambda reply: request_status_reply_hander(reply, install_page, upgrade_page, uninstall_page), error_handler=lambda e:handle_dbus_error("request_status", e), ) list_message = [] list_message.append(_("Failed to refresh package lists.")) list_message.append(_('Try again')) list_message.append(lambda:global_event.emit('start-update-list')) global_event.emit("show-message", list_message, 0) global_event.emit('update-list-finish') global_event.emit("hide-update-list-dialog") elif signal_type == "parse-download-error": (pkg_name, action_type) = action_content if action_type == ACTION_INSTALL: install_page.download_parse_failed(pkg_name) global_event.emit("show-message", _("Problem occurred when analyzing dependencies for %s. Installation aborted.") % pkg_name) elif action_type == ACTION_UPGRADE: upgrade_page.download_parse_failed(pkg_name) global_event.emit("show-message", _("Problem occurred when analyzing dependencies for %s. Upgrade aborted.") % pkg_name) elif signal_type == "pkg-not-in-cache": pkg_name = action_content list_message = [] list_message.append(_('The requested package \"%s\" was not found in the package list.') % pkg_name) list_message.append(_('Refresh package lists again and try again.')) list_message.append(lambda:global_event.emit('start-update-list')) global_event.emit("show-message", list_message, 0) except Exception, e: traceback.print_exc(file=sys.stdout) print "Message Handler Error:", e
def take_turn(self, team, prompt=None, move=None): """ Interact with player to facilitate moves, capture data and identify/store information common to all potential moves. Also includes optional param to specify a prompt to run automatically or a move object (for interface from external scripts). """ global LOG self.turns += 1 self.current_team = team occupied, our_team, their_team = self.get_occupied() validated, found_issue = False, False if self.check: user_feedback = shout(team + ' team in check', print_output=False, return_output=True) else: user_feedback = None # repeat prompt until a valid move is given... while not validated: # skip set up if a move object is passed in... if move: piece, up, right = move.piece, move.up, move.right else: if not prompt: print(self.board.draw_board()) if user_feedback: print(user_feedback + '\n') prompt = input("[" + team + " move] >> ") piece, up, right, hold_move, user_feedback = \ self.__parse_prompt(prompt, our_team) if not hold_move: # create object for move, this evaluates potential issues etc. move = Move(piece, up, right, occupied, our_team, their_team) if move: if move.possible: validated = True else: user_feedback = move.invalid_reason move = None # clear ready for next loop prompt = None # clear ready for next loop # noinspection PyUnboundLocalVariable self.__process_move(piece, move, up, right, occupied, our_team, their_team) # log state of game if self.logging: LOG = '\n'.join([LOG, self.__to_json()]) # wrap up if done... if self.checkmate or self.turns >= 200: if self.turns >= self.turn_limit: shout('{0} moves, lets call it a draw'.format(self.turns)) elif self.checkmate: move.checkmate = True shout('game over, {0} team wins'.format(self.current_team)) if self.logging: write_log(LOG)