def flag(self, message, fl): ''' flag v.s. message: 1 <-> 1 ''' if isinstance(message, snstype.Message): #digest = message.digest_pyobj msg_id = message.msg_id else: msg_id = message cur = self.con.cursor() ret = False try: cur.execute( ''' UPDATE msg SET flag=? WHERE id=? ''', (fl, msg_id)) self.con.commit() ret = True except Exception, e: logger.warning("Catch exception: %s", e)
def select_samples(message): candidates = {} null_msg = [] for m in message['seen_list']: if len(m.tags) >= 1: #candidates.append(m) candidates[m.msg_id] = m else: null_msg.append(m) logger.warning(str(len(message['seen_list']))) # Sample same number of null tag messages prob = float(len(candidates)) / (len(message['seen_list']) - len(candidates)) if prob > 1.0: prob = 1.0 print "Selecting null message probability: %.3f" % (prob) for m in null_msg: if random.random() < prob: #m.tags = {"null": 1} # All tag id is greater than or equal to 1. # 0 is reserved as "null" tag if the user has not defined one. m.tags = {0: 1} #candidates.append(m) candidates[m.msg_id] = m print "Total %d samples extracted" % (len(candidates)) return candidates
def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object self.msgMapping = { "Email":1, "FacebookFeed":2, "RSS":3, "RSS2RW":4, "RSSSummary":5, "RenrenBlog":6, "RenrenFeed": 7, "RenrenPhoto": 8, "RenrenShare":9, "RenrenStatus":10, "RenrenStatusDirect": 11, "SQLite":12, "SinaWeiboBase":13, "SinaWeiboStatus":14, "SinaWeiboWapStatus":15, "TencentWeiboStatus":16, "TwitterStatus":17, "InstagramFeed":18, "DoubanFeed":19} self.platform_num = 19 self.inputnum = 10 self.condLast = time.time() try: self.queue_conf = json.loads(open('conf' + delimeter + 'queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def config(): info = {} (k, sp, ap, q) = token_to_user(request.get_cookie('token')) pr = get_preference(k) if sp is None: return {"info": {}, "sp": {}, "ap": {}, "q": {}} for ch in sp: info[ch] = sp[ch].jsonconf info[ch]['expire_after'] = int(sp[ch].expire_after()) info[ch]['is_authed'] = sp[ch].is_authed() info[ch]['need_auth'] = sp[ch].need_auth() con_user = sqlite3.connect("." + delimeter + "User" + delimeter + str(k) + delimeter + "srfe_queue.db", check_same_thread = False) con_user.isolation_level = None cur_user = con_user.cursor() r = cur_user.execute("SELECT platform, toggle FROM msg_toggle WHERE toggle <> 2") for m in r: for ch in info.values(): logger.warning(str(ch)) logger.warning(str(m[0])) if ch["platform"] == m[0]: ch["toggle"] = m[1] r = cur_user.execute("SELECT id, username, platform, toggle FROM user_toggle") bu = {} for m in r: bu[str(m[0])] = { "id": m[0], "username": m[1], "platform": m[2], "toggle": m[3] } return {"info": info, "sp": sp, "ap": ap, "q": q, "pr":pr, "bu": bu}
def wrapper_check_login(*al, **ad): logger.warning(str(user)) token = request.get_cookie("token") if not check_token(token): redirect('/login') else: return func(*al, **ad)
def run(self): while (self.keep_running): try: self.queue.input() logger.debug("Invoke input() on queue") time.sleep(INPUT_GAP) except Exception as e: logger.warning("Catch Exception in InputThread: %s", e)
def run(self): while (self.keep_running): try: self.queue.input() logger.debug("Invoke input() on queue") time.sleep(INPUT_GAP) except Exception as e: logger.warning("Catch Exception in InputThread: %s", e)
class Feature(object): """docstring for Feature""" env = { "dir_conf": "./conf", "dir_kdb": "./kdb", } feature_extractors = [] try: awjson = json.loads(open('conf/autoweight.json').read()) features = awjson['features'] import plugin for f in features: module_name = f[0] class_name = f[1] #print module_name #print class_name mo = __import__("ranking.plugin.%s" % module_name, fromlist=["ranking.plugin"]) cl = getattr(mo, class_name) feature_extractors.append(cl(env)) #TODO: # Make the dynamic import method better (more standard ways). # The current import method is borrowed from: # http://stackoverflow.com/questions/301134/dynamic-module-import-in-python # It just works. #cl = __import__("plugin.%s.%s" % (module_name, class_name), fromlist=["plugin.%s" % module_name]) #cl = getattr(getattr(plugin, module_name), class_name) except IOError: logger.warning('No "conf/autoweight.json"!') except KeyError: logger.warning('No "features" defined"!') def __init__(self): super(Feature, self).__init__() @staticmethod def extract(msg): ''' Feature extraction. It will extract features to a dict and store in "msg.feature". msg: an snstype.Message object ''' if not isinstance(msg, snstype.Message): logger.warning( "Cannot extract feature for non snstype.Message object") return # Add all kinds of features msg.feature = {} for fe in Feature.feature_extractors: fe.add_features(msg)
def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def __init__(self, snspocket=None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def config_preference_add(): k = token_to_user_key(request.get_cookie('token')) for (w,l) in zip(request.forms.getall("winner"), request.forms.getall("loser")): if not [w,l] in user_sp[k]["pr"]["preference"] and w != l: user_sp[str(k)]["pr"]["preference"].append([w, l]) try: json.dump(user_sp[str(k)]["pr"], open('." + delimeter + "User" + delimeter + "' + str(k) + '" + delimeter + "conf" + delimeter + "autoweight.json', "w")) except Exception as e: logger.warning("Catch Exception in InputThread: %s", e) return "Preference has been added!"
def load_weight(self, fn = None): if fn is None: fn = 'conf/weights.json' try: self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight) except IOError: logger.warning("No '%s' weights config file, use empty setting.", fn) self.feature_weight = {} self.feature_name = self.feature_weight.keys()
def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object #self.__mount_default_home_timeline_count() #self.queue_conf = json.load(open('conf/queue.json', 'r')) #self.feature_weight = self.queue_conf['feature_weight'] try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def load_weight(self, fn=None): if fn is None: fn = 'conf/weights.json' try: self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight) except IOError: logger.warning("No '%s' weights config file, use empty setting.", fn) self.feature_weight = {} self.feature_name = self.feature_weight.keys()
def get_unseen_count(self): cur = self.con.cursor() r = cur.execute(''' SELECT count(*) FROM msg WHERE flag='unseen' ''') try: return r.next()[0] except Exception, e: logger.warning("Catch Exception: %s", e) return -1
def user_toggle(self, msg_id): cur = self.con.cursor() try: r = cur.execute(''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id, )) str_obj = r.next()[0] message = self._str2pyobj(str_obj) except Exception, e: logger.warning("Catch exception: %s", e) return
def get_unseen_count(self): cur = self.con.cursor() r = cur.execute(''' SELECT count(*) FROM msg WHERE flag='unseen' ''') try: return r.next()[0] except Exception, e: logger.warning("Catch Exception: %s", e) return -1
def run(self): #webbrowser.open("http://127.0.0.1:8080/login") while (self.keep_running): self.queue.input() try: logger.debug("Invoke input() on queue") os.chdir("." + delimeter + "User" + delimeter + str(self.user_id)) p = os.getcwd() self.sp.save_config(fn_channel= p + delimeter + "conf" + delimeter + "channel.json",fn_pocket= p + delimeter + "conf" + delimeter + "pocket.json") os.chdir(".." + delimeter + ".." + delimeter) time.sleep(INPUT_GAP) except Exception as e: logger.warning("Catch Exception in InputThread: %s", e)
def home_timeline(): (k, sp, ap, q) = token_to_user(request.get_cookie('token')) (sl, tl) = q.output(q.inputnum) logger.warning(str(len(tl))) meta = { "unseen_count": q.get_unseen_count() } if len(sl) > 0: q.condLast = sl[-1].parsed.time mp = {} for (s,v) in sp.items(): mp[s] = q.msgMapping[v.platform] return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
def reweight_all(self, younger_than = 86400): begin = self.time() cur = self.con.cursor() try: latest_time = int(self.time() - younger_than) r = cur.execute(''' SELECT id from msg WHERE time >= ? ''', (latest_time, )) for m in r: self.reweight(m[0]) except Exception, e: logger.warning("Catch exception: %s", e) return False
def sql(self, query_string): cur = self.con.cursor() try: r = cur.execute(query_string) logger.debug("SQL query string: %s", query_string) message_list = snstype.MessageList() for m in r: obj = self._str2pyobj(m[1]) obj.msg_id = m[0] message_list.append(obj) return message_list except Exception, e: logger.warning("Catch exception when executing '%s': %s", query_string, e) return snstype.MessageList()
def auth_second(): (k, sp, ap, q) = token_to_user(request.get_cookie('token')) op = "auth_second for %s" % (ap.current_channel) qs = request.query_string # For compatibility with lower level interface. # The snsbase parses code from the whole url. ap.code_url = "http://snsapi.snsapi/auth/second/auth?%s" % qs sp[ap.current_channel].auth_second() logger.warning("c") os.chdir("." + delimeter + "User" + delimeter + str(k)) sp[ap.current_channel].save_token() os.chdir(".." + delimeter + ".." + delimeter) logger.warning("d") result = "done: %s" % qs return {'operation': op, 'result': result}
def reweight_all(self, younger_than=86400): begin = self.time() cur = self.con.cursor() try: latest_time = int(self.time() - younger_than) r = cur.execute( ''' SELECT id from msg WHERE time >= ? ''', (latest_time, )) for m in r: self.reweight(m[0]) except Exception, e: logger.warning("Catch exception: %s", e) return False
def update_from_console(t, *al, **ad): ''' A wrapper function to deal with user input from console. String input from console is in console encoding. We must first cast it to unicode, which is the standard across SNSAPI. ''' if isinstance(t, str): return sp.update(console_input(t), *al, **ad) elif isinstance(t, snstype.Message): return sp.update(t, *al, **ad) else: logger.warning("unknown type: %s", type(t))
def update_from_console(t, *al, **ad): ''' A wrapper function to deal with user input from console. String input from console is in console encoding. We must first cast it to unicode, which is the standard across SNSAPI. ''' if isinstance(t, str): return sp.update(console_input(t), *al, **ad) elif isinstance(t, snstype.Message): return sp.update(t, *al, **ad) else: logger.warning("unknown type: %s", type(t))
def sql(self, query_string): cur = self.con.cursor() try: r = cur.execute(query_string) logger.debug("SQL query string: %s", query_string) message_list = snstype.MessageList() for m in r: obj = self._str2pyobj(m[1]) obj.msg_id = m[0] message_list.append(obj) return message_list except Exception, e: logger.warning("Catch exception when executing '%s': %s", query_string, e) return snstype.MessageList()
def ranked_timeline(): (k, sp, ap, q) = token_to_user(request.get_cookie('token')) (sl, tl) = q.output_ranked(q.inputnum, 86400) meta = { "unseen_count": q.get_unseen_count() } logger.warning(str(q.condLast)) if len(sl) > 0: if str(sl[-1]["weight"]) == "0.0": q.condLast = 0.0001 else: q.condLast = sl[-1]["weight"] logger.warning(str(q.condLast)) mp = {} for (s,v) in sp.items(): mp[s] = q.msgMapping[v.platform] return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
def extract(msg): """ Feature extraction. It will extract features to a dict and store in "msg.feature". msg: an snstype.Message object """ if not isinstance(msg, snstype.Message): logger.warning("Cannot extract feature for non snstype.Message object") return # Add all kinds of features msg.feature = {} for fe in Feature.feature_extractors: fe.add_features(msg)
def reweight(self, msg_id): cur = self.con.cursor() try: r = cur.execute(''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id,)) m = self._str2pyobj(list(r)[0][0]) w = self._weight_feature(m) t = int(self.time()) r = cur.execute(''' UPDATE msg SET weight=?,weight_time=? WHERE id=? ''', (w, t, msg_id)) except Exception, e: logger.warning("Catch exception: %s", e)
def reply(self, msg_id, comment, channel = None): cur = self.con.cursor() try: r = cur.execute(''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id, )) str_obj = r.next()[0] message = self._str2pyobj(str_obj) result = self.sp.reply(message, comment, channel) self.log('[reply]%s;%s;%s' % (msg_id, result)) return result except Exception, e: logger.warning("Catch exception: %s", e) return {}
def extract(msg): ''' Feature extraction. It will extract features to a dict and store in "msg.feature". msg: an snstype.Message object ''' if not isinstance(msg, snstype.Message): logger.warning( "Cannot extract feature for non snstype.Message object") return # Add all kinds of features msg.feature = {} for fe in Feature.feature_extractors: fe.add_features(msg)
def forward(self, msg_id, comment): cur = self.con.cursor() try: r = cur.execute( ''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id, )) str_obj = r.next()[0] message = self._str2pyobj(str_obj) result = self.sp.forward(message, comment) self.log('[forward]%s;%s;%s' % (msg_id, result, comment)) return result except Exception, e: logger.warning("Catch exception: %s", e) return {}
def sql(self, condition): cur = self.con.cursor() try: # We trust the client string. This software is intended for personal use. qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition r = cur.execute(qs) logger.debug("SQL query string: %s", qs) message_list = snstype.MessageList() for m in r: obj = self._str2pyobj(m[1]) obj.msg_id = m[0] message_list.append(obj) return message_list except Exception, e: logger.warning("Catch exception when executing '%s': %s", condition, e) return snstype.MessageList()
def sql(self, condition): cur = self.con.cursor() try: # We trust the client string. This software is intended for personal use. qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition r = cur.execute(qs) logger.debug("SQL query string: %s", qs) message_list = snstype.MessageList() for m in r: obj = self._str2pyobj(m[1]) obj.msg_id = m[0] message_list.append(obj) return message_list except Exception, e: logger.warning("Catch exception when executing '%s': %s", condition, e) return snstype.MessageList()
def reweight(self, msg_id): cur = self.con.cursor() try: r = cur.execute( ''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id, )) m = self._str2pyobj(list(r)[0][0]) w = self._weight_feature(m) t = int(self.time()) r = cur.execute( ''' UPDATE msg SET weight=?,weight_time=? WHERE id=? ''', (w, t, msg_id)) except Exception, e: logger.warning("Catch exception: %s", e)
def unlike(self, msg_id, channel = None): cur = self.con.cursor() try: r = cur.execute(''' SELECT pyobj FROM msg WHERE id=? ''', (msg_id, )) str_obj = r.next()[0] message = self._str2pyobj(str_obj) result = self.sp.unlike(message, channel) self.log('[unlike]%s;%s' % (msg_id, result)) if result: cur.execute("UPDATE msg SET like = 0 WHERE id=?", (msg_id, )) return result except Exception, e: logger.warning("Catch exception: %s", e) return False
def tag(self, message, tg): ''' flag v.s. message: * <-> * ''' if isinstance(message, snstype.Message): msg_id = message.msg_id else: msg_id = message cur = self.con.cursor() ret = False try: cur.execute(''' INSERT INTO msg_tag(msg_id, tag_id) VALUES (?,?) ''', (msg_id, tg)) self.con.commit() ret = True except Exception, e: logger.warning("Catch exception: %s", e)
def tag(self, message, tg): ''' flag v.s. message: * <-> * ''' if isinstance(message, snstype.Message): msg_id = message.msg_id else: msg_id = message cur = self.con.cursor() ret = False try: cur.execute( ''' INSERT INTO msg_tag(msg_id, tag_id) VALUES (?,?) ''', (msg_id, tg)) self.con.commit() ret = True except Exception, e: logger.warning("Catch exception: %s", e)
def flag(self, message, fl): ''' flag v.s. message: 1 <-> 1 ''' if isinstance(message, snstype.Message): #digest = message.digest_pyobj msg_id = message.msg_id else: msg_id = message cur = self.con.cursor() ret = False try: cur.execute(''' UPDATE msg SET flag=? WHERE id=? ''', (fl, msg_id)) self.con.commit() ret = True except Exception, e: logger.warning("Catch exception: %s", e)
def home_timeline_ajax(): (k, sp, ap, q) = token_to_user(request.get_cookie('token')) if q.condLast > 1000: cond = "msg.time < " + str(q.condLast) (sl, tl) = q.output(count = q.inputnum, condition = cond) else: q.condLast = 0.0001 if q.condLast == 0.0 else q.condLast cond = "msg.weight <= " + str(q.condLast) logger.warning(cond) (sl, tl) = q.output_ranked(count = q.inputnum, younger_than = 86400, condition = cond) meta = { "unseen_count": q.get_unseen_count() } mp = {} logger.warning(str(len(sl))) if len(sl) > 0: q.condLast = sl[-1].parsed.time if q.condLast > 1000 else sl[-1].weight q.condLast = 0.0001 if q.condLast == 0.0 else q.condLast logger.warning(str(q.condLast)) for (s,v) in sp.items(): mp[s] = q.msgMapping[v.platform] return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
sp.load_config() for c in sp.values(): c.request_url = lambda url: ap.request_url(url) c.fetch_code = lambda : ap.fetch_code() c.auth() srfe = Bottle() q = SRFEQueue(sp) q.connect() q.refresh_tags() try: jsonconf = json.load(open('conf/srfe.json', 'r')) except IOError: logger.warning("Do not find conf/srfe.json, use defaults") jsonconf = { "cookie_sign_key": "You had better set your own key in the config!!!!", "username": "******", "password": "******" } INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch logger.debug("INPUT_GAP: %s", INPUT_GAP) class InputThread(threading.Thread): def __init__(self, queue): super(InputThread, self).__init__() self.queue = queue self.keep_running = True
def toggle_close(self, platform): cur = self.con.cursor() try: cur.execute("UPDATE msg_toggle SET toggle = 0 WHERE platform = ?", (platform,)) except: logger.warning(e)
sp.load_config() for c in sp.values(): c.request_url = lambda url: ap.request_url(url) c.fetch_code = lambda : ap.fetch_code() c.auth() srfe = Bottle() q = SRFEQueue(sp) q.connect() q.refresh_tags() try: jsonconf = json.load(open('conf/srfe.json', 'r')) except IOError: logger.warning("Do not find conf/srfe.json, use defaults") jsonconf = { "cookie_sign_key": "You had better set your own key in the config!!!!", "username": "******", "password": "******" } INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch logger.debug("INPUT_GAP: %s", INPUT_GAP) class InputThread(threading.Thread): def __init__(self, queue): super(InputThread, self).__init__() self.queue = queue self.keep_running = True
# -*- coding: utf-8 -*- import sys sys.path.append('snsapi') from functools import wraps import json import snsapi from snsapi.snspocket import SNSPocket from snsapi.snslog import SNSLog as logger from lbucket import * try: _wauto_conf = json.loads(open('conf/wauto.json').read()) except Exception, e: logger.warning("Load conf error: %s. Use default", e) _wauto_conf = {} _wauto_conf['priority'] = { 'update': 5, 'home_timeline': 4, 'forward': 3, 'reply': 3 } logger.debug("conf: %s", _wauto_conf) ''' Make the invokation from Python interpreter more convenient. Use synchronous calls. ''' def _dummy_decorator_generator(*args, **kwargs): def _dummy_decorator(func):
# -*- coding: utf-8 -*- import sys sys.path.append('snsapi') from functools import wraps import json import snsapi from snsapi.snspocket import SNSPocket from snsapi.snslog import SNSLog as logger from lbucket import * try: _wauto_conf = json.loads(open('conf/wauto.json').read()) except Exception, e: logger.warning("Load conf error: %s. Use default", e) _wauto_conf = {} _wauto_conf['priority'] = { 'update': 5, 'home_timeline': 4, 'forward': 3, 'reply': 3 } logger.debug("conf: %s", _wauto_conf) ''' Make the invokation from Python interpreter more convenient. Use synchronous calls. ''' def _dummy_decorator_generator(*args, **kwargs):
def _inqueue(self, message): cur = self.con.cursor() try: # Deduplicate # Explain the problem of the following two methods for future reference: # 1. digest = self._digest_pyobj(message) # Python object are hashed to different values even the SNS message # fields are all the same. # 2. digest = message.digest_parsed() # I forget what is the problem.. I should have noted before. digest = message.digest() #logger.debug("message pyobj digest '%s'", digest) r = cur.execute( ''' SELECT digest FROM msg WHERE digest = ? ''', (digest, )) if len(list(r)) > 0: #logger.debug("message '%s' already exists", digest) return False else: logger.debug("message '%s' is new", digest) #TODO: # This is temporary solution for object digestion. # # For our Message object, the following evaluates to False!! # Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o))) # # To perform deduplication and further refer to this message, # we store the calculated digestion as an attribute of the message. # Note however, after this operation the digest of 'message' will not # be the valued stored therein! This is common problem in such mechanism, # e.g. UDP checksum. Developers should have this in mind. message.digest_pyobj = self._digest_pyobj(message) cur.execute(''' INSERT INTO msg( time , text , userid , username , mid , platform , digest , digest_parsed , digest_pyobj , parsed , pyobj , flag , weight , weight_time ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ''', (\ message.parsed.time,\ message.parsed.text,\ message.parsed.userid,\ message.parsed.username,\ str(message.ID),\ message.platform,\ message.digest(),\ message.digest_parsed(),\ #self._digest_pyobj(message),\ message.digest_pyobj,\ message.dump_parsed(),\ self._pyobj2str(message),\ "unseen", self._weight_feature(message), int(time.time()) )) return True except Exception, e: logger.warning("failed: %s", str(e)) #print message #raise e return False
def _inqueue(self, message): cur = self.con.cursor() try: # Deduplicate # Explain the problem of the following two methods for future reference: # 1. digest = self._digest_pyobj(message) # Python object are hashed to different values even the SNS message # fields are all the same. # 2. digest = message.digest_parsed() # I forget what is the problem.. I should have noted before. digest = message.digest() #logger.debug("message pyobj digest '%s'", digest) r = cur.execute(''' SELECT digest FROM msg WHERE digest = ? ''', (digest, )) if len(list(r)) > 0: #logger.warning("message '%s' already exists", str(message)) return False else: #logger.warning("message '%s' is new", str(message)) self._hook_new_message(self, message) #TODO: # This is temporary solution for object digestion. # # For our Message object, the following evaluates to False!! # Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o))) # # To perform deduplication and further refer to this message, # we store the calculated digestion as an attribute of the message. # Note however, after this operation the digest of 'message' will not # be the valued stored therein! This is common problem in such mechanism, # e.g. UDP checksum. Developers should have this in mind. message.digest_pyobj = self._digest_pyobj(message) try: platform_id = self.msgMapping[message.platform] except: self.platform_num = self.paltform_num + 1 platform_id = self.platform_num self.msgMapping[message.platform] = self.platform_num cur.execute('''INSERT INTO msg_toggle (platform_id, platform) VALUES (null, message.platform)''') if not hasattr(message.parsed, "liked") or message.parsed.liked == False: like_flag = 0 else: like_flag = 1 cur.execute(""" UPDATE msg_toggle SET toggle = 1 WHERE platform_id = ? """, (platform_id,)) cur.execute(''' INSERT INTO msg( time , text , userid , username , mid , platform_id, digest , digest_parsed , digest_pyobj , parsed , pyobj , flag , weight , weight_time, like ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ''', (\ message.parsed.time,\ message.parsed.text,\ message.parsed.userid,\ message.parsed.username,\ str(message.ID),\ platform_id,\ message.digest(),\ message.digest_parsed(),\ #self._digest_pyobj(message),\ message.digest_pyobj,\ message.dump_parsed(),\ self._pyobj2str(message),\ "unseen", self._weight_feature(message), int(self.time()), like_flag )) return True except Exception, e: logger.warning("failed: %s", str(e)) #print message #raise e return False