def flag(self, message, fl):
'''
flag v.s. message: 1 <-> 1
'''
if isinstance(message, snstype.Message):
#digest = message.digest_pyobj
msg_id = message.msg_id
else:
msg_id = message
cur = self.con.cursor()
ret = False
try:
cur.execute(
'''
UPDATE msg
SET flag=?
WHERE id=?
''', (fl, msg_id))
self.con.commit()
ret = True
except Exception, e:
logger.warning("Catch exception: %s", e)
def select_samples(message):
candidates = {}
null_msg = []
for m in message['seen_list']:
if len(m.tags) >= 1:
#candidates.append(m)
candidates[m.msg_id] = m
else:
null_msg.append(m)
logger.warning(str(len(message['seen_list'])))
# Sample same number of null tag messages
prob = float(len(candidates)) / (len(message['seen_list']) - len(candidates))
if prob > 1.0:
prob = 1.0
print "Selecting null message probability: %.3f" % (prob)
for m in null_msg:
if random.random() < prob:
#m.tags = {"null": 1}
# All tag id is greater than or equal to 1.
# 0 is reserved as "null" tag if the user has not defined one.
m.tags = {0: 1}
#candidates.append(m)
candidates[m.msg_id] = m
print "Total %d samples extracted" % (len(candidates))
return candidates
def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
self.msgMapping = {
"Email":1,
"FacebookFeed":2,
"RSS":3,
"RSS2RW":4,
"RSSSummary":5,
"RenrenBlog":6,
"RenrenFeed": 7,
"RenrenPhoto": 8,
"RenrenShare":9,
"RenrenStatus":10,
"RenrenStatusDirect": 11,
"SQLite":12,
"SinaWeiboBase":13,
"SinaWeiboStatus":14,
"SinaWeiboWapStatus":15,
"TencentWeiboStatus":16,
"TwitterStatus":17,
"InstagramFeed":18,
"DoubanFeed":19}
self.platform_num = 19
self.inputnum = 10
self.condLast = time.time()
try:
self.queue_conf = json.loads(open('conf' + delimeter + 'queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def config():
info = {}
(k, sp, ap, q) = token_to_user(request.get_cookie('token'))
pr = get_preference(k)
if sp is None:
return {"info": {}, "sp": {}, "ap": {}, "q": {}}
for ch in sp:
info[ch] = sp[ch].jsonconf
info[ch]['expire_after'] = int(sp[ch].expire_after())
info[ch]['is_authed'] = sp[ch].is_authed()
info[ch]['need_auth'] = sp[ch].need_auth()
con_user = sqlite3.connect("." + delimeter + "User" + delimeter + str(k) + delimeter + "srfe_queue.db", check_same_thread = False)
con_user.isolation_level = None
cur_user = con_user.cursor()
r = cur_user.execute("SELECT platform, toggle FROM msg_toggle WHERE toggle <> 2")
for m in r:
for ch in info.values():
logger.warning(str(ch))
logger.warning(str(m[0]))
if ch["platform"] == m[0]:
ch["toggle"] = m[1]
r = cur_user.execute("SELECT id, username, platform, toggle FROM user_toggle")
bu = {}
for m in r:
bu[str(m[0])] = {
"id": m[0],
"username": m[1],
"platform": m[2],
"toggle": m[3]
}
return {"info": info, "sp": sp, "ap": ap, "q": q, "pr":pr, "bu": bu}
def wrapper_check_login(*al, **ad):
logger.warning(str(user))
token = request.get_cookie("token")
if not check_token(token):
redirect('/login')
else:
return func(*al, **ad)
def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
class Feature(object):
"""docstring for Feature"""
env = {
"dir_conf": "./conf",
"dir_kdb": "./kdb",
}
feature_extractors = []
try:
awjson = json.loads(open('conf/autoweight.json').read())
features = awjson['features']
import plugin
for f in features:
module_name = f[0]
class_name = f[1]
#print module_name
#print class_name
mo = __import__("ranking.plugin.%s" % module_name,
fromlist=["ranking.plugin"])
cl = getattr(mo, class_name)
feature_extractors.append(cl(env))
#TODO:
# Make the dynamic import method better (more standard ways).
# The current import method is borrowed from:
# http://stackoverflow.com/questions/301134/dynamic-module-import-in-python
# It just works.
#cl = __import__("plugin.%s.%s" % (module_name, class_name), fromlist=["plugin.%s" % module_name])
#cl = getattr(getattr(plugin, module_name), class_name)
except IOError:
logger.warning('No "conf/autoweight.json"!')
except KeyError:
logger.warning('No "features" defined"!')
def __init__(self):
super(Feature, self).__init__()
@staticmethod
def extract(msg):
'''
Feature extraction.
It will extract features to a dict and store in "msg.feature".
msg: an snstype.Message object
'''
if not isinstance(msg, snstype.Message):
logger.warning(
"Cannot extract feature for non snstype.Message object")
return
# Add all kinds of features
msg.feature = {}
for fe in Feature.feature_extractors:
fe.add_features(msg)
def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def __init__(self, snspocket=None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def config_preference_add():
k = token_to_user_key(request.get_cookie('token'))
for (w,l) in zip(request.forms.getall("winner"), request.forms.getall("loser")):
if not [w,l] in user_sp[k]["pr"]["preference"] and w != l:
user_sp[str(k)]["pr"]["preference"].append([w, l])
try:
json.dump(user_sp[str(k)]["pr"], open('." + delimeter + "User" + delimeter + "' + str(k) + '" + delimeter + "conf" + delimeter + "autoweight.json', "w"))
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
return "Preference has been added!"
def load_weight(self, fn = None):
if fn is None:
fn = 'conf/weights.json'
try:
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
except IOError:
logger.warning("No '%s' weights config file, use empty setting.", fn)
self.feature_weight = {}
self.feature_name = self.feature_weight.keys()
def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
#self.__mount_default_home_timeline_count()
#self.queue_conf = json.load(open('conf/queue.json', 'r'))
#self.feature_weight = self.queue_conf['feature_weight']
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def load_weight(self, fn=None):
if fn is None:
fn = 'conf/weights.json'
try:
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
except IOError:
logger.warning("No '%s' weights config file, use empty setting.",
fn)
self.feature_weight = {}
self.feature_name = self.feature_weight.keys()
def get_unseen_count(self):
cur = self.con.cursor()
r = cur.execute('''
SELECT count(*) FROM msg
WHERE flag='unseen'
''')
try:
return r.next()[0]
except Exception, e:
logger.warning("Catch Exception: %s", e)
return -1
def user_toggle(self, msg_id):
cur = self.con.cursor()
try:
r = cur.execute('''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id, ))
str_obj = r.next()[0]
message = self._str2pyobj(str_obj)
except Exception, e:
logger.warning("Catch exception: %s", e)
return
def get_unseen_count(self):
cur = self.con.cursor()
r = cur.execute('''
SELECT count(*) FROM msg
WHERE flag='unseen'
''')
try:
return r.next()[0]
except Exception, e:
logger.warning("Catch Exception: %s", e)
return -1
def run(self):
#webbrowser.open("http://127.0.0.1:8080/login")
while (self.keep_running):
self.queue.input()
try:
logger.debug("Invoke input() on queue")
os.chdir("." + delimeter + "User" + delimeter + str(self.user_id))
p = os.getcwd()
self.sp.save_config(fn_channel= p + delimeter + "conf" + delimeter + "channel.json",fn_pocket= p + delimeter + "conf" + delimeter + "pocket.json")
os.chdir(".." + delimeter + ".." + delimeter)
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def home_timeline():
(k, sp, ap, q) = token_to_user(request.get_cookie('token'))
(sl, tl) = q.output(q.inputnum)
logger.warning(str(len(tl)))
meta = {
"unseen_count": q.get_unseen_count()
}
if len(sl) > 0:
q.condLast = sl[-1].parsed.time
mp = {}
for (s,v) in sp.items():
mp[s] = q.msgMapping[v.platform]
return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
def reweight_all(self, younger_than = 86400):
begin = self.time()
cur = self.con.cursor()
try:
latest_time = int(self.time() - younger_than)
r = cur.execute('''
SELECT id from msg
WHERE time >= ?
''', (latest_time, ))
for m in r:
self.reweight(m[0])
except Exception, e:
logger.warning("Catch exception: %s", e)
return False
def sql(self, query_string):
cur = self.con.cursor()
try:
r = cur.execute(query_string)
logger.debug("SQL query string: %s", query_string)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s", query_string, e)
return snstype.MessageList()
def auth_second():
(k, sp, ap, q) = token_to_user(request.get_cookie('token'))
op = "auth_second for %s" % (ap.current_channel)
qs = request.query_string
# For compatibility with lower level interface.
# The snsbase parses code from the whole url.
ap.code_url = "http://snsapi.snsapi/auth/second/auth?%s" % qs
sp[ap.current_channel].auth_second()
logger.warning("c")
os.chdir("." + delimeter + "User" + delimeter + str(k))
sp[ap.current_channel].save_token()
os.chdir(".." + delimeter + ".." + delimeter)
logger.warning("d")
result = "done: %s" % qs
return {'operation': op, 'result': result}
def reweight_all(self, younger_than=86400):
begin = self.time()
cur = self.con.cursor()
try:
latest_time = int(self.time() - younger_than)
r = cur.execute(
'''
SELECT id from msg
WHERE time >= ?
''', (latest_time, ))
for m in r:
self.reweight(m[0])
except Exception, e:
logger.warning("Catch exception: %s", e)
return False
def update_from_console(t, *al, **ad):
'''
A wrapper function to deal with user input from console.
String input from console is in console encoding. We must
first cast it to unicode, which is the standard across
SNSAPI.
'''
if isinstance(t, str):
return sp.update(console_input(t), *al, **ad)
elif isinstance(t, snstype.Message):
return sp.update(t, *al, **ad)
else:
logger.warning("unknown type: %s", type(t))
def update_from_console(t, *al, **ad):
'''
A wrapper function to deal with user input from console.
String input from console is in console encoding. We must
first cast it to unicode, which is the standard across
SNSAPI.
'''
if isinstance(t, str):
return sp.update(console_input(t), *al, **ad)
elif isinstance(t, snstype.Message):
return sp.update(t, *al, **ad)
else:
logger.warning("unknown type: %s", type(t))
def sql(self, query_string):
cur = self.con.cursor()
try:
r = cur.execute(query_string)
logger.debug("SQL query string: %s", query_string)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s",
query_string, e)
return snstype.MessageList()
def ranked_timeline():
(k, sp, ap, q) = token_to_user(request.get_cookie('token'))
(sl, tl) = q.output_ranked(q.inputnum, 86400)
meta = {
"unseen_count": q.get_unseen_count()
}
logger.warning(str(q.condLast))
if len(sl) > 0:
if str(sl[-1]["weight"]) == "0.0":
q.condLast = 0.0001
else:
q.condLast = sl[-1]["weight"]
logger.warning(str(q.condLast))
mp = {}
for (s,v) in sp.items():
mp[s] = q.msgMapping[v.platform]
return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
def extract(msg):
"""
Feature extraction.
It will extract features to a dict and store in "msg.feature".
msg: an snstype.Message object
"""
if not isinstance(msg, snstype.Message):
logger.warning("Cannot extract feature for non snstype.Message object")
return
# Add all kinds of features
msg.feature = {}
for fe in Feature.feature_extractors:
fe.add_features(msg)
def reweight(self, msg_id):
cur = self.con.cursor()
try:
r = cur.execute('''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id,))
m = self._str2pyobj(list(r)[0][0])
w = self._weight_feature(m)
t = int(self.time())
r = cur.execute('''
UPDATE msg
SET weight=?,weight_time=?
WHERE id=?
''', (w, t, msg_id))
except Exception, e:
logger.warning("Catch exception: %s", e)
def reply(self, msg_id, comment, channel = None):
cur = self.con.cursor()
try:
r = cur.execute('''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id, ))
str_obj = r.next()[0]
message = self._str2pyobj(str_obj)
result = self.sp.reply(message, comment, channel)
self.log('[reply]%s;%s;%s' % (msg_id, result))
return result
except Exception, e:
logger.warning("Catch exception: %s", e)
return {}
def extract(msg):
'''
Feature extraction.
It will extract features to a dict and store in "msg.feature".
msg: an snstype.Message object
'''
if not isinstance(msg, snstype.Message):
logger.warning(
"Cannot extract feature for non snstype.Message object")
return
# Add all kinds of features
msg.feature = {}
for fe in Feature.feature_extractors:
fe.add_features(msg)
def forward(self, msg_id, comment):
cur = self.con.cursor()
try:
r = cur.execute(
'''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id, ))
str_obj = r.next()[0]
message = self._str2pyobj(str_obj)
result = self.sp.forward(message, comment)
self.log('[forward]%s;%s;%s' % (msg_id, result, comment))
return result
except Exception, e:
logger.warning("Catch exception: %s", e)
return {}
def sql(self, condition):
cur = self.con.cursor()
try:
# We trust the client string. This software is intended for personal use.
qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition
r = cur.execute(qs)
logger.debug("SQL query string: %s", qs)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s", condition, e)
return snstype.MessageList()
def sql(self, condition):
cur = self.con.cursor()
try:
# We trust the client string. This software is intended for personal use.
qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition
r = cur.execute(qs)
logger.debug("SQL query string: %s", qs)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s",
condition, e)
return snstype.MessageList()
def reweight(self, msg_id):
cur = self.con.cursor()
try:
r = cur.execute(
'''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id, ))
m = self._str2pyobj(list(r)[0][0])
w = self._weight_feature(m)
t = int(self.time())
r = cur.execute(
'''
UPDATE msg
SET weight=?,weight_time=?
WHERE id=?
''', (w, t, msg_id))
except Exception, e:
logger.warning("Catch exception: %s", e)
def unlike(self, msg_id, channel = None):
cur = self.con.cursor()
try:
r = cur.execute('''
SELECT pyobj FROM msg
WHERE id=?
''', (msg_id, ))
str_obj = r.next()[0]
message = self._str2pyobj(str_obj)
result = self.sp.unlike(message, channel)
self.log('[unlike]%s;%s' % (msg_id, result))
if result:
cur.execute("UPDATE msg SET like = 0 WHERE id=?", (msg_id, ))
return result
except Exception, e:
logger.warning("Catch exception: %s", e)
return False
def tag(self, message, tg):
'''
flag v.s. message: * <-> *
'''
if isinstance(message, snstype.Message):
msg_id = message.msg_id
else:
msg_id = message
cur = self.con.cursor()
ret = False
try:
cur.execute('''
INSERT INTO msg_tag(msg_id, tag_id)
VALUES (?,?)
''', (msg_id, tg))
self.con.commit()
ret = True
except Exception, e:
logger.warning("Catch exception: %s", e)
def tag(self, message, tg):
'''
flag v.s. message: * <-> *
'''
if isinstance(message, snstype.Message):
msg_id = message.msg_id
else:
msg_id = message
cur = self.con.cursor()
ret = False
try:
cur.execute(
'''
INSERT INTO msg_tag(msg_id, tag_id)
VALUES (?,?)
''', (msg_id, tg))
self.con.commit()
ret = True
except Exception, e:
logger.warning("Catch exception: %s", e)
def flag(self, message, fl):
'''
flag v.s. message: 1 <-> 1
'''
if isinstance(message, snstype.Message):
#digest = message.digest_pyobj
msg_id = message.msg_id
else:
msg_id = message
cur = self.con.cursor()
ret = False
try:
cur.execute('''
UPDATE msg
SET flag=?
WHERE id=?
''', (fl, msg_id))
self.con.commit()
ret = True
except Exception, e:
logger.warning("Catch exception: %s", e)
def home_timeline_ajax():
(k, sp, ap, q) = token_to_user(request.get_cookie('token'))
if q.condLast > 1000:
cond = "msg.time < " + str(q.condLast)
(sl, tl) = q.output(count = q.inputnum, condition = cond)
else:
q.condLast = 0.0001 if q.condLast == 0.0 else q.condLast
cond = "msg.weight <= " + str(q.condLast)
logger.warning(cond)
(sl, tl) = q.output_ranked(count = q.inputnum, younger_than = 86400, condition = cond)
meta = {
"unseen_count": q.get_unseen_count()
}
mp = {}
logger.warning(str(len(sl)))
if len(sl) > 0:
q.condLast = sl[-1].parsed.time if q.condLast > 1000 else sl[-1].weight
q.condLast = 0.0001 if q.condLast == 0.0 else q.condLast
logger.warning(str(q.condLast))
for (s,v) in sp.items():
mp[s] = q.msgMapping[v.platform]
return {'sl': sl, 'mp': mp, 'tl': tl, 'snsapi_utils': snsapi_utils, 'tags': q.get_tags(), 'meta': meta, "token" : request.get_cookie('token')}
sp.load_config()
for c in sp.values():
c.request_url = lambda url: ap.request_url(url)
c.fetch_code = lambda : ap.fetch_code()
c.auth()
srfe = Bottle()
q = SRFEQueue(sp)
q.connect()
q.refresh_tags()
try:
jsonconf = json.load(open('conf/srfe.json', 'r'))
except IOError:
logger.warning("Do not find conf/srfe.json, use defaults")
jsonconf = {
"cookie_sign_key": "You had better set your own key in the config!!!!",
"username": "******",
"password": "******"
}
INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch
logger.debug("INPUT_GAP: %s", INPUT_GAP)
class InputThread(threading.Thread):
def __init__(self, queue):
super(InputThread, self).__init__()
self.queue = queue
self.keep_running = True
def toggle_close(self, platform):
cur = self.con.cursor()
try:
cur.execute("UPDATE msg_toggle SET toggle = 0 WHERE platform = ?", (platform,))
except:
logger.warning(e)
sp.load_config()
for c in sp.values():
c.request_url = lambda url: ap.request_url(url)
c.fetch_code = lambda : ap.fetch_code()
c.auth()
srfe = Bottle()
q = SRFEQueue(sp)
q.connect()
q.refresh_tags()
try:
jsonconf = json.load(open('conf/srfe.json', 'r'))
except IOError:
logger.warning("Do not find conf/srfe.json, use defaults")
jsonconf = {
"cookie_sign_key": "You had better set your own key in the config!!!!",
"username": "******",
"password": "******"
}
INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch
logger.debug("INPUT_GAP: %s", INPUT_GAP)
class InputThread(threading.Thread):
def __init__(self, queue):
super(InputThread, self).__init__()
self.queue = queue
self.keep_running = True
# -*- coding: utf-8 -*-
import sys
sys.path.append('snsapi')
from functools import wraps
import json
import snsapi
from snsapi.snspocket import SNSPocket
from snsapi.snslog import SNSLog as logger
from lbucket import *
try:
_wauto_conf = json.loads(open('conf/wauto.json').read())
except Exception, e:
logger.warning("Load conf error: %s. Use default", e)
_wauto_conf = {}
_wauto_conf['priority'] = {
'update': 5,
'home_timeline': 4,
'forward': 3,
'reply': 3
}
logger.debug("conf: %s", _wauto_conf)
'''
Make the invokation from Python interpreter more convenient.
Use synchronous calls.
'''
def _dummy_decorator_generator(*args, **kwargs):
def _dummy_decorator(func):
# -*- coding: utf-8 -*-
import sys
sys.path.append('snsapi')
from functools import wraps
import json
import snsapi
from snsapi.snspocket import SNSPocket
from snsapi.snslog import SNSLog as logger
from lbucket import *
try:
_wauto_conf = json.loads(open('conf/wauto.json').read())
except Exception, e:
logger.warning("Load conf error: %s. Use default", e)
_wauto_conf = {}
_wauto_conf['priority'] = {
'update': 5,
'home_timeline': 4,
'forward': 3,
'reply': 3
}
logger.debug("conf: %s", _wauto_conf)
'''
Make the invokation from Python interpreter more convenient.
Use synchronous calls.
'''
def _dummy_decorator_generator(*args, **kwargs):
def _inqueue(self, message):
cur = self.con.cursor()
try:
# Deduplicate
# Explain the problem of the following two methods for future reference:
# 1. digest = self._digest_pyobj(message)
# Python object are hashed to different values even the SNS message
# fields are all the same.
# 2. digest = message.digest_parsed()
# I forget what is the problem.. I should have noted before.
digest = message.digest()
#logger.debug("message pyobj digest '%s'", digest)
r = cur.execute(
'''
SELECT digest FROM msg
WHERE digest = ?
''', (digest, ))
if len(list(r)) > 0:
#logger.debug("message '%s' already exists", digest)
return False
else:
logger.debug("message '%s' is new", digest)
#TODO:
# This is temporary solution for object digestion.
#
# For our Message object, the following evaluates to False!!
# Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o)))
#
# To perform deduplication and further refer to this message,
# we store the calculated digestion as an attribute of the message.
# Note however, after this operation the digest of 'message' will not
# be the valued stored therein! This is common problem in such mechanism,
# e.g. UDP checksum. Developers should have this in mind.
message.digest_pyobj = self._digest_pyobj(message)
cur.execute('''
INSERT INTO msg(
time ,
text ,
userid ,
username ,
mid ,
platform ,
digest ,
digest_parsed ,
digest_pyobj ,
parsed ,
pyobj ,
flag ,
weight ,
weight_time
)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
''', (\
message.parsed.time,\
message.parsed.text,\
message.parsed.userid,\
message.parsed.username,\
str(message.ID),\
message.platform,\
message.digest(),\
message.digest_parsed(),\
#self._digest_pyobj(message),\
message.digest_pyobj,\
message.dump_parsed(),\
self._pyobj2str(message),\
"unseen",
self._weight_feature(message),
int(time.time())
))
return True
except Exception, e:
logger.warning("failed: %s", str(e))
#print message
#raise e
return False
def _inqueue(self, message):
cur = self.con.cursor()
try:
# Deduplicate
# Explain the problem of the following two methods for future reference:
# 1. digest = self._digest_pyobj(message)
# Python object are hashed to different values even the SNS message
# fields are all the same.
# 2. digest = message.digest_parsed()
# I forget what is the problem.. I should have noted before.
digest = message.digest()
#logger.debug("message pyobj digest '%s'", digest)
r = cur.execute('''
SELECT digest FROM msg
WHERE digest = ?
''', (digest, ))
if len(list(r)) > 0:
#logger.warning("message '%s' already exists", str(message))
return False
else:
#logger.warning("message '%s' is new", str(message))
self._hook_new_message(self, message)
#TODO:
# This is temporary solution for object digestion.
#
# For our Message object, the following evaluates to False!!
# Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o)))
#
# To perform deduplication and further refer to this message,
# we store the calculated digestion as an attribute of the message.
# Note however, after this operation the digest of 'message' will not
# be the valued stored therein! This is common problem in such mechanism,
# e.g. UDP checksum. Developers should have this in mind.
message.digest_pyobj = self._digest_pyobj(message)
try:
platform_id = self.msgMapping[message.platform]
except:
self.platform_num = self.paltform_num + 1
platform_id = self.platform_num
self.msgMapping[message.platform] = self.platform_num
cur.execute('''INSERT INTO msg_toggle (platform_id, platform) VALUES (null, message.platform)''')
if not hasattr(message.parsed, "liked") or message.parsed.liked == False:
like_flag = 0
else:
like_flag = 1
cur.execute("""
UPDATE msg_toggle SET toggle = 1 WHERE platform_id = ?
""", (platform_id,))
cur.execute('''
INSERT INTO msg(
time ,
text ,
userid ,
username ,
mid ,
platform_id,
digest ,
digest_parsed ,
digest_pyobj ,
parsed ,
pyobj ,
flag ,
weight ,
weight_time,
like
)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
''', (\
message.parsed.time,\
message.parsed.text,\
message.parsed.userid,\
message.parsed.username,\
str(message.ID),\
platform_id,\
message.digest(),\
message.digest_parsed(),\
#self._digest_pyobj(message),\
message.digest_pyobj,\
message.dump_parsed(),\
self._pyobj2str(message),\
"unseen",
self._weight_feature(message),
int(self.time()),
like_flag
))
return True
except Exception, e:
logger.warning("failed: %s", str(e))
#print message
#raise e
return False
