def add_history(cls, jid, to_jid, content): m = cls.get_one(jid) cls.db[const.MEMBER].update( {"_id": m._id}, {"$set": {"last_say": now()}}) cls.db[const.HISTORY].insert( {"from_member": cls.db.ref(const.MEMBER, m._id), "to_member": to_jid, "content": content, "pubdate": now()})
def modify_nick(cls, jid, nick): """ 修改成员昵称 Arguments: `jid` - jid `nick` - 新昵称 Return: False // 昵称已存在 True // 更改昵称成功 """ m = cls.get_one(jid) if not m: return False if m: exists = cls.get_with_nick(nick) if exists: return False m.nick = nick m.last_change = now() cls.set_info( jid, "change_nick_times", int(cls.get_info(jid, "change_nick_times", 0).value) + 1) try: session.commit() except: session.rollback() return True
def combine_klg(self, uid, weibomsg): if (weibomsg['run'] != ''): text = weibomsg['mc'] + "//@" + weibomsg['run'] + weibomsg['rmc'] else: text = weibomsg['mc'] klg_list = utility.match_klg(text) if (len(klg_list) > 0): # 存在某个词条 try: insert_batch = [] update_mt = '' for klg in klg_list: insert_batch.append([weibomsg['mid'], klg]) update_mt += str(klg) + ',' update_mt = update_mt.rstrip(',') self.db.insert( "INSERT IGNORE INTO weibomsg (mid,uid,un,sn,iu,mc,murl,srn,iurl,rc,cc,pt,nc) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [ weibomsg['mid'], uid, weibomsg['un'], weibomsg['un'], weibomsg['iu'], text, weibomsg['page'], weibomsg['srn'], weibomsg['pu'], weibomsg['rc'], weibomsg['cc'], weibomsg['pt'], weibomsg['nc'] ]) self.db.insert_many( "INSERT IGNORE INTO abbre_weibomsg (mid,abrid) VALUES(%s, %s)", insert_batch) self.db.update( "UPDATE `abbreviation` SET mt=%s WHERE abrid IN (" + update_mt + ")", [utility.now()]) except Exception, e: logging.exception("%s 绑定数据库失败: " + str(e), weibomsg['mid'])
def login(loginName,password): user = _is_repeat_login(loginName,password) if user: return user c = utility.get_cursor() c.execute("SELECT ID,NickName,Sex,birthdayYear,certfState,Password FROM u_user WHERE password=? AND nickname=?",(password,loginName)) rows = c.fetchall() if len(rows) == 0: print("SELECT ID,NickName,Sex,birthdayYear,certfState,Password FROM u_user WHERE password='******' AND nickname='%s'"%(password,loginName)) utility.write_log(-1,loginName+"登陆失败",0) return None r = rows[0] user = session_data() user.user_id = r[0] user.nickname = r[1] user.sex = r[2] user.session_id = make_session_id() user.age = utility.now().year - r[3] user.certf_state = r[4] user.pwd = r[5] user.ip = utility.get_ip() global g_session_data g_session_data[user.session_id] = user utility.write_log(user.user_id,"登陆成功",1) return user
def update_class_result(class_result: pd.DataFrame, result: pd.DataFrame, feature_list: List[str], model_list: List[str]): """ 用result更新class_result :param class_result: 需要更新的数据 :param result: 用来更新的数据 :param feature_list: 特征集合 :param model_list: 模型集合 :return: None """ result['update_time'] = now() if class_result.empty: pass else: if_exist = (class_result['model_name'].isin(model_list)) \ & (class_result['feature_name'].isin(feature_list)) if if_exist.sum(): exist_index = if_exist[if_exist].index class_result.drop(index=exist_index, inplace=True) class_result = class_result.append(result, sort=False) class_result.sort_values(['feature_name', 'model_name'], inplace=True) class_result.reset_index(drop=True, inplace=True) return class_result
def url_index2(): s = None save = False if bottle.request.query.sex: #直接访问 i = int(bottle.request.query.sex) if i != 1 and i != 0: i = 0 s = session.visit(i) elif bottle.request.forms.session: #注册用户访问 s = session.try_get(bottle.request.forms.session) if bottle.request.forms.save: save = True else: #从保存cookies的index页面中直接redirect过来的情况 session_id = utility.get_session_id() if session_id != "-1": s = session.try_get(session_id) if not s: bottle.redirect("/") timeout = utility.now() + datetime.timedelta(days = 365) if save: bottle.response.set_cookie("uname",s.nickname,secret="abc123",expires=timeout) bottle.response.set_cookie("pwd",s.pwd,secret="abc123",expires=timeout) bottle.response.set_cookie("loginstate","yes",expires=timeout) #标识用户已经登陆过 bottle.response.set_cookie('session',s.session_id) d = session.get_dist(s) return bottle.template('index2', d)
def fday(idchat): dbh = DBHelper() result = dbh.select_idchat('data', idchat) current_date = utility.now() current_data = current_date.split('-')[1] elapsed_time = 0 for r in result: if r[2].split('-')[1] == current_data: elapsed_day = utility.difference(r[2], r[3]) elapsed_time += elapsed_day result = dbh.select_idchat('turno', idchat) if len(result) == 1: elapsed_time = elapsed_time + utility.difference( result[0][2], utility.now()) return 'Ore lavorate oggi: ' + utility.str_duration(elapsed_time)
def model_validate(self, feature_list: List[str], model_list: List[str]): """ 模型验证 :param feature_list: 特征集合 :param model_list: 模型集合 :return: None """ groups = self.groups cv_predict_result = pd.DataFrame() for feature_name in feature_list: feature = self.features[feature_name] data = self.source_train[feature] target = self.source_train['target'] for model_name in model_list: LOG.info(f'开始验证 {feature_name} {model_name}') model = self.search_model(feature_name, model_name) pool = Pool(5) result = [] for group in groups.keys(): group_result = pool.apply_async( self.k_fold_validation, args=(feature_name, model_name, groups, group, data, target, model)) result.append(group_result) pool.close() result = [group_result.get() for group_result in result] result = pd.DataFrame(result) self.append_validate_result( feature_name=feature_name, name=model_name, score_train=result['train_error'].mean(), score_validation=result['validation_error'].mean(), mean_train_time=result['run_time'].mean(), param=json.dumps(model.get_params()), update_time=now()) col = [ 'feature_name', 'model_name', 'group', 'train_predict', 'validation_predict' ] cv_predict_result = cv_predict_result.append(result[col]) LOG.info('验证结束') self.update_cv_predict(feature_list, model_list, cv_predict_result)
def add_history(cls, jid, to_jid, content): m = cls.get_one(jid) m.last_say = now() if m.history: m.history.append(History(to_jid, content)) else: m.history = [History(to_jid, content)] try: session.commit() except: session.rollback()
def fit(self, x_train, y_train, x_test=[], y_test=[], is_classification=False): xp = np if self.gpu_flag >= 0: cuda.check_cuda_available() cuda.get_device(self.gpu_flag).use() xp = cuda.cupy self.model.to_gpu() if self.log_path != "" and not os.path.isfile(self.log_path): print "log_path: "+self.log_path utility.writeText(self.log_path, "a", "datetime,epoch,train_loss,test_loss,test_accuracy\n") for epoch in xrange(self.epoch): train_size = len(x_train) train_loss = 0. test_loss = 0. test_accuracy = 0. indexes = np.random.permutation(train_size) for i in xrange(0, train_size, self.batch_size): x = Variable(xp.asarray(x_train[indexes[i:i+self.batch_size]])) t = Variable(xp.asarray(y_train[indexes[i:i+self.batch_size]])) self.model.zerograds() loss = self.loss_function(self.forward(x), t) loss.backward() self.optimizer.update() train_loss += loss.data * len(t.data) train_loss /= train_size test_size = len(x_test) for i in xrange(0, test_size, self.batch_size): x = Variable(xp.asarray(x_test[i:i+self.batch_size])) t = Variable(xp.asarray(y_test[i:i+self.batch_size])) test_loss += float(self.loss_function(self.forward(x), t).data) * len(t.data) if is_classification: test_accuracy += float(F.accuracy(self.forward(x), t).data) * len(t.data) if test_size != 0: test_loss /= test_size test_accuracy /= test_size if epoch > 1 and self.log_path != "": utility.writeText(self.log_path,"a", "%s,%d,%f,%f,%f\n"% (utility.now(), epoch, train_loss, test_loss, test_accuracy)) if self.gpu_flag >= 0: self.model.to_cpu() if self.export_path != "": pickle.dump(self.model, open(self.export_path, 'wb'), -1) return self
def set_global_info(cls, key, value): """ 设置全局选项 Arguments: `key` - 选项键 `value` - 选项值 """ info, f, _ = cls._get_info(key=key, is_global=True) if f: cls.db[const.INFO].update({"_id": info._id}, {"$set": {"value": value}}) else: cls.db[const.INFO].insert({"key": key, "value": value, "pubdate": now(), "is_global": True}) return info
def fpunch(idchat): current_d = utility.now() dbh = DBHelper() result = dbh.select_idchat('turno', idchat) if len(result) == 0: dbh.insert_idchat('turno', ['idchat', 'start'], [idchat, current_d]) return "Inizio: " + str(current_d) else: dbh.update_time('turno', result[0][0], current_d) elapsed_time = utility.difference(result[0][2], current_d) dbh.update_hours(idchat, elapsed_time) dbh.delete_id('turno', result[0][0]) return "Fine: " + str( current_d) + "\n" + "Durata: " + utility.str_duration(elapsed_time)
def update_mac_record(user, mac, duration, agent, device, profile): ''' agent : user agents device: client device's type (IPhone|IPad|Android|Windows NT) ''' is_update = False expired = utility.now('%Y-%m-%d %H:%M:%S', hours=duration) record = store.get_user_mac_record(user, mac) if record: is_update = True try: store.update_mac_record(user, mac, expired, agent, device, profile['ssid'], is_update) except IntegrityError: # duplicate entry pass
def set_info(cls, jid, key, value): """ 设置成员选项 Arguments: `jid` - jid `key` - 选项键 `value` - 选项值 """ info, f, m = cls._get_info(jid, key) if f: cls.db[const.INFO].update({"_id": info._id}, {"$set": {"value": value}}) else: cls.db[const.INFO].insert({"key": key, "value": value, "is_global": False, "pubdate": now(), "mid": m._id}) return info
def add(cls, jid, nick=None, show=None): """ 添加一个成员 Arguments: `jid` - 成员jid `nick` - 昵称 `show` - stanze.show """ if cls.get_one(jid): return if not nick: nick = get_email(jid).split("@")[0] doc = {"email": get_email(jid), "nick": nick, "isonline": True, "join_date": now()} mid = cls.db[const.MEMBER].insert(doc) cls.db[const.STATUS].insert({"mid": mid, "statustext": show, "resource": jid.resource, "status": const.ONLINE}) return cls.get_one(jid)
def register(nick,pwd,sex,age): assert int(sex) <= 1 age = int(age) try: now = utility.now() db = utility.get_db() c = db.cursor() c.execute("INSERT INTO u_user (NickName,Sex,Password,CreateDate,BirthdayYear,CertfState)VALUES(?,?,?,?,?,?)", (nick,int(sex),pwd,now.strftime("%Y-%m-%d %H:%M:%S"),now.year - age,0)) db.commit() #不这样做取不到user_id user = session.login(nick,pwd) c.execute("INSERT INTO u_profile(ID,EditDate)VALUES(?,?)",(user.user_id,now)) utility.write_log(user.user_id,"注册成功",1,False) db.commit() return user except Exception as err: utility.write_log(-1,nick+"注册失败",0) traceback.print_exc() return None;
def update_online_record(user, mac, ap_mac, ssid, status='alive'): ''' 1. update stop record 2. if ap change, create new records, change id 3. stop: remove record from ONLINE_RECORDS ''' key = '{}_{}'.format(user, mac) if key in ONLINE_RECORDS: _id, pre_ap_mac = ONLINE_RECORDS[key] stop = utility.now() # update records _id = store.update_online_record(_id, stop, user, mac, ap_mac, ssid, pre_ap_mac) if ap_mac != pre_ap_mac: # update new record ONLINE_RECORDS[key] = (_id, ap_mac) else: # create new record add_online_record(user, mac, ap_mac, ssid) if status=='stop': # clear cache ONLINE_RECORDS.pop(key, '')
def modify_nick(cls, jid, nick): """ 修改成员昵称 Arguments: `jid` - jid `nick` - 新昵称 Return: False // 昵称已存在 True // 更改昵称成功 """ m = cls.get_one(jid) if not m: return False if m: exists = cls.get_with_nick(nick) if exists: return False m.nick = nick m.last_change = now() cls.set_info(jid, "change_nick_times", int(cls.get_info(jid, "change_nick_times", 0).value) + 1) try: session.commit() except: session.rollback() return True
def modify_nick(cls, jid, nick): """ 修改成员昵称 Arguments: `jid` - jid `nick` - 新昵称 Return: False // 昵称已存在 True // 更改昵称成功 """ m = cls.get_one(jid) if not m: return False if m: exists = cls.get_with_nick(nick) if exists: return False cls.db[const.MEMBER].update({"_id": m._id}, {"$set": {"nick": nick, "last_change": now()}, "$push": {"used_nick": nick}}) cls.set_info(jid, const.INFO_CHANGE_NICK_TIMES, int(cls.get_info(jid, const.INFO_CHANGE_NICK_TIMES, 0).value) + 1) return True
def model_test(self, feature_list: List[str], model_list: List[str]): """ 模型测试 :param feature_list: 特征集合 :param model_list: 模型集合 :return: None """ for feature_name in feature_list: feature = self.features[feature_name] train_set = self.source_train[feature] target = self.source_train['target'] test_set = self.test_set[feature] for model_name in model_list: LOG.info(f'开始测试集预测 {feature_name} {model_name}') model = self.search_model(feature_name, model_name) model.fit(train_set, target) time_now = now() self.append_test_result(feature_name=feature_name, name=model_name, score=None, param=json.dumps(model.get_params()), update_time=time_now) predict = model.predict(test_set) self.test_out(predict, time_now, model_name, feature_name) LOG.info('测试集预测结束')
def merge_test(self, feature_list: List[str], merge_list: List[str], model_list: List[str], stacking_model: str): """ 融合测试 :param feature_list: 特征集合 :param merge_list: 融合集合 :param model_list: 融合需要的模型集合 :param stacking_model: stacking要用的模型名 :return: """ for feature_name in feature_list: feature = self.features[feature_name] data = self.source_train[feature] target = self.source_train['target'] test_set = self.test_set[feature] for merge_name in merge_list: LOG.info(f'开始测试集预测 {feature_name} {merge_name}') if merge_name == 'Mean': result = [] for model_name in model_list: model = self.search_model(feature_name, model_name) model.fit(data, target) result.append(model.predict(test_set)) predict = np.array(result).mean(axis=0) name = merge_name elif merge_name == 'Stacking': (model, validation_predict, validation_target, test_predict) = self.k_fold_stacking( feature_name=feature_name, model_list=model_list, groups=self.groups, test_set=test_set, stacking_model=stacking_model) predict = model.predict(test_predict) name = merge_name + '_' + stacking_model else: raise Exception(f'merge_list参数输入错误 {merge_list}') time_now = now() self.append_test_result(feature_name=feature_name, name=name, score=None, param=json.dumps(model_list), update_time=time_now) self.test_out(predict, time_now, name, feature_name) LOG.info('测试集预测结束')
def merge_validate(self, feature_list: List[str], merge_list: List[str], model_list: List[str], stacking_model: str): """ 融合验证 :param feature_list: 特征集合 :param merge_list: 融合集合 :param model_list: 融合需要的模型集合 :param stacking_model: stacking要用的模型名 :return: """ groups = self.groups for feature_name in feature_list: feature = self.features[feature_name] data = self.source_train[feature] target = self.source_train['target'] for merge_name in merge_list: LOG.info(f'开始验证 {feature_name} {merge_name}') result = pd.DataFrame() if merge_name == 'Mean': cv_predict = self.cv_predict choose = (cv_predict['feature_name'] == feature_name) \ & (cv_predict['model_name'].isin(model_list)) cv_predict_choose = cv_predict.loc[choose] for group in groups.keys(): group_predict = cv_predict_choose.loc[ cv_predict_choose['group'] == group] start_time = time.time() for item in ['train', 'validation']: item_predict = np.array( list(group_predict[item + '_predict'].values)) mean_predict = item_predict.mean(axis=0) true_value = target[groups[group][item]] score = mean_squared_error(true_value, mean_predict) result.loc[group, item] = score end_time = time.time() result.loc[group, 'run_time'] = end_time - start_time name = merge_name elif merge_name == 'Stacking': for group in groups.keys(): inner_groups = self.stacking_groups[group].copy() test_index = inner_groups['test'] test_set = data.loc[test_index] test_target = target[test_index] del inner_groups['test'] start_time = time.time() (model, validation_predict, validation_target, test_predict) = self.k_fold_stacking( feature_name=feature_name, model_list=model_list, groups=inner_groups, test_set=test_set, stacking_model=stacking_model) end_time = time.time() predict_validation = model.predict(validation_predict) predict_test = model.predict(test_predict) train_score = mean_squared_error( predict_validation, validation_target) validation_score = mean_squared_error( predict_test, test_target) result.loc[group, 'train'] = train_score result.loc[group, 'validation'] = validation_score result.loc[group, 'run_time'] = end_time - start_time name = merge_name + '_' + stacking_model else: raise Exception(f'merge_list参数输入错误 {merge_list}') self.append_validate_result( feature_name=feature_name, name=name, score_train=result['train'].mean(), score_validation=result['validation'].mean(), mean_train_time=result['run_time'].mean(), param=json.dumps(model_list), update_time=now()) LOG.info('验证结束')
def combine_klg(self, uid, weibomsg): if(weibomsg['run'] != ''): text = weibomsg['mc'] + "//@" + weibomsg['run'] + weibomsg['rmc']; else: text = weibomsg['mc']; klg_list = utility.match_klg(text); if( len(klg_list) > 0 ): # 存在某个词条 try: insert_batch = []; update_mt = ''; for klg in klg_list: insert_batch.append( [weibomsg['mid'], klg] ); update_mt += str(klg) + ','; update_mt = update_mt.rstrip(','); self.db.insert("INSERT IGNORE INTO weibomsg (mid,uid,un,sn,iu,mc,murl,srn,iurl,rc,cc,pt,nc) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [weibomsg['mid'], uid, weibomsg['un'], weibomsg['un'], weibomsg['iu'], text, weibomsg['page'], weibomsg['srn'], weibomsg['pu'], weibomsg['rc'], weibomsg['cc'], weibomsg['pt'], weibomsg['nc']]); self.db.insert_many("INSERT IGNORE INTO abbre_weibomsg (mid,abrid) VALUES(%s, %s)", insert_batch); self.db.update("UPDATE `abbreviation` SET mt=%s WHERE abrid IN (" + update_mt + ")", [utility.now()]); except Exception,e: logging.exception("%s 绑定数据库失败: " + str(e), weibomsg['mid']);
def fit(self, x_train, y_train, x_test=[], y_test=[], is_classification=False): xp = np if self.gpu_flag >= 0: cuda.check_cuda_available() cuda.get_device(self.gpu_flag).use() xp = cuda.cupy self.model.to_gpu() if self.log_path != "" and not os.path.isfile(self.log_path): print "log_path: " + self.log_path utility.writeText( self.log_path, "a", "datetime,epoch,train_loss,test_loss,test_accuracy\n") for epoch in xrange(self.epoch): train_size = len(x_train) train_loss = 0. test_loss = 0. test_accuracy = 0. indexes = np.random.permutation(train_size) for i in xrange(0, train_size, self.batch_size): x = Variable( xp.asarray(x_train[indexes[i:i + self.batch_size]])) t = Variable( xp.asarray(y_train[indexes[i:i + self.batch_size]])) self.model.zerograds() loss = self.loss_function(self.forward(x), t) loss.backward() self.optimizer.update() train_loss += loss.data * len(t.data) train_loss /= train_size test_size = len(x_test) for i in xrange(0, test_size, self.batch_size): x = Variable(xp.asarray(x_test[i:i + self.batch_size])) t = Variable(xp.asarray(y_test[i:i + self.batch_size])) test_loss += float( self.loss_function(self.forward(x), t).data) * len(t.data) if is_classification: test_accuracy += float( F.accuracy(self.forward(x), t).data) * len(t.data) if test_size != 0: test_loss /= test_size test_accuracy /= test_size if epoch > 1 and self.log_path != "": utility.writeText( self.log_path, "a", "%s,%d,%f,%f,%f\n" % (utility.now(), epoch, train_loss, test_loss, test_accuracy)) if self.gpu_flag >= 0: self.model.to_cpu() if self.export_path != "": pickle.dump(self.model, open(self.export_path, 'wb'), -1) return self
return -1; def update(self, query, value, db=None): self.__connect(db) self.cursor.execute(query, value) self.conn.commit() count = self.__getRC(); self.__close() return count def delete(self, query, value, db=None): self.__connect(db) self.cursor.execute(query, value) self.conn.commit() count = self.__getRC(); self.__close() return count def get_last_query(self): return self.cursor._last_executed; if __name__ == '__main__': db = DBHelper('localhost','root','tangbin//','ecshop'); #print db.insert_many("INSERT INTO ecs_brand (brand_name) VALUES (%s)", [['test7'],['test8']]); #db.delete("DELETE FROM ecs_brand WHERE brand_name=%s", ['test9']); #print db.get_last_query(); #res = db.select_one("SELECT count(brand_name) FROM ecs_brand WHERE brand_name='%s'"%'诺基亚'); #print res[0]; update_mt = '175,599' db.update("UPDATE `abbreviation` SET mt=%s WHERE abrid IN (" + update_mt + ")", [utility.now()]);
def add_honor(cls, jid, value, typ, item, desc): m = cls.get_with_nick(jid) doc = {"getdate": now(), "date": now(), "type": typ, "desc": desc, "mid": m._id, "item": item, "value": value} cls.db[const.HONOR].insert(doc)
def get_today_rps(cls): now = now() starttime = datetime(now.year, now.month, now.day) endtime = datetime(now.year, now.month, now.day, 23, 59, 59) return sorted(cls.get_all_rps(starttime, endtime), key=lambda x: x["value"])
def __init__(self, jid, nick = None): self.nick = nick if nick else jid.local self.email = get_email(jid) self.join_date = now()
def __init__(self, to_member, content): self.to_member = get_email(to_member) self.content = content self.pubdate = now()
def set_today_rp(cls, jid, rp): cls.set_info(jid, const.INFO_RP, rp) cls.set_info(jid, const.INFO_RP_DATE, time.time()) cls.db[const.MEMBER].update({"email": get_email(jid)}, {"$push": {"rps": {"value": rp, "date": now()}}})
self.__connect(db) self.cursor.execute(query, value) self.conn.commit() count = self.__getRC() self.__close() return count def delete(self, query, value, db=None): self.__connect(db) self.cursor.execute(query, value) self.conn.commit() count = self.__getRC() self.__close() return count def get_last_query(self): return self.cursor._last_executed if __name__ == '__main__': db = DBHelper('localhost', 'root', 'tangbin//', 'ecshop') #print db.insert_many("INSERT INTO ecs_brand (brand_name) VALUES (%s)", [['test7'],['test8']]); #db.delete("DELETE FROM ecs_brand WHERE brand_name=%s", ['test9']); #print db.get_last_query(); #res = db.select_one("SELECT count(brand_name) FROM ecs_brand WHERE brand_name='%s'"%'诺基亚'); #print res[0]; update_mt = '175,599' db.update( "UPDATE `abbreviation` SET mt=%s WHERE abrid IN (" + update_mt + ")", [utility.now()])
def __init__(self, key, value, is_global = False): self.key = key self.value = value self.is_global = 1 if is_global else 0 self.pubdate = now()