def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def tag_add(self, name):
cur = self.con.cursor()
r = cur.execute('''
INSERT INTO tag(name, visible)
VALUES(?, ?)
''', (name, 1))
logger.debug("Add tag %s", name)
self.refresh_tags()
def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def tag_toggle(self, tag_id):
cur_visible = self.tags_all[tag_id]['visible']
cur = self.con.cursor()
r = cur.execute('''
UPDATE tag
SET visible=?
WHERE id=?
''', (1 - cur_visible, tag_id))
logger.debug("Set tag %d to visibility %d", tag_id, 1 - cur_visible)
def rand_execute(prob, func):
if prob >= 1.0:
func()
else:
if random.random() < prob:
logger.debug('func execute. prob: %f', prob)
return func()
else:
logger.debug('func do not execute. prob: %f', prob)
def tag_add(self, name):
cur = self.con.cursor()
r = cur.execute(
'''
INSERT INTO tag(name, visible)
VALUES(?, ?)
''', (name, 1))
logger.debug("Add tag %s", name)
self.refresh_tags()
def hook_new_message(q, msg):
mymsg = False
sp = q.sp
for ch in sp.values():
if 'user_name' in ch.jsonconf:
user_name = ch.jsonconf['user_name']
if user_name == msg.parsed.username:
mymsg = True
if mymsg:
logger.debug('distribute message: %s', msg)
distribute.update(msg)
def tag_toggle(self, tag_id):
cur_visible = self.tags_all[tag_id]['visible']
cur = self.con.cursor()
r = cur.execute(
'''
UPDATE tag
SET visible=?
WHERE id=?
''', (1 - cur_visible, tag_id))
logger.debug("Set tag %d to visibility %d", tag_id, 1 - cur_visible)
self.refresh_tags()
def hook_new_message(q, msg):
mymsg = False
sp = q.sp
for ch in sp.values():
if 'user_name' in ch.jsonconf:
user_name = ch.jsonconf['user_name']
if user_name == msg.parsed.username:
mymsg = True
if mymsg:
logger.debug('distribute message: %s', msg)
distribute.update(msg)
def run(self):
#webbrowser.open("http://127.0.0.1:8080/login")
while (self.keep_running):
self.queue.input()
try:
logger.debug("Invoke input() on queue")
os.chdir("." + delimeter + "User" + delimeter + str(self.user_id))
p = os.getcwd()
self.sp.save_config(fn_channel= p + delimeter + "conf" + delimeter + "channel.json",fn_pocket= p + delimeter + "conf" + delimeter + "pocket.json")
os.chdir(".." + delimeter + ".." + delimeter)
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def sql(self, query_string):
cur = self.con.cursor()
try:
r = cur.execute(query_string)
logger.debug("SQL query string: %s", query_string)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s", query_string, e)
return snstype.MessageList()
def sql(self, query_string):
cur = self.con.cursor()
try:
r = cur.execute(query_string)
logger.debug("SQL query string: %s", query_string)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s",
query_string, e)
return snstype.MessageList()
def sql(self, condition):
cur = self.con.cursor()
try:
# We trust the client string. This software is intended for personal use.
qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition
r = cur.execute(qs)
logger.debug("SQL query string: %s", qs)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s", condition, e)
return snstype.MessageList()
def tag_add(self, name):
cur = self.con.cursor()
r = cur.execute('''
SELECT id FROM tag WHERE name=?
''', (name, ))
for t in cur:
if type(t[0]) == int:
return "Duplicate tag found."
r = cur.execute('''
INSERT INTO tag(name, visible)
VALUES(?, ?)
''', (name, 1))
logger.debug("Add tag %s", name)
self.refresh_tags()
r = cur.execute('''
SELECT id FROM tag WHERE name=?
''', (name, ))
for t in cur:
return t[0]
def sql(self, condition):
cur = self.con.cursor()
try:
# We trust the client string. This software is intended for personal use.
qs = "SELECT DISTINCT msg.id,msg.pyobj FROM msg,msg_tag WHERE %s" % condition
r = cur.execute(qs)
logger.debug("SQL query string: %s", qs)
message_list = snstype.MessageList()
for m in r:
obj = self._str2pyobj(m[1])
obj.msg_id = m[0]
message_list.append(obj)
return message_list
except Exception, e:
logger.warning("Catch exception when executing '%s': %s",
condition, e)
return snstype.MessageList()
from snsapi.snspocket import SNSPocket
from snsapi.snslog import SNSLog as logger
from lbucket import *
try:
_wauto_conf = json.loads(open('conf/wauto.json').read())
except Exception, e:
logger.warning("Load conf error: %s. Use default", e)
_wauto_conf = {}
_wauto_conf['priority'] = {
'update': 5,
'home_timeline': 4,
'forward': 3,
'reply': 3
}
logger.debug("conf: %s", _wauto_conf)
'''
Make the invokation from Python interpreter more convenient.
Use synchronous calls.
'''
def _dummy_decorator_generator(*args, **kwargs):
def _dummy_decorator(func):
return func
return _dummy_decorator
if __name__ == '__main__':
rate_limit = _dummy_decorator_generator
from snsapi.snspocket import SNSPocket
from snsapi.snslog import SNSLog as logger
from lbucket import *
try:
_wauto_conf = json.loads(open('conf/wauto.json').read())
except Exception, e:
logger.warning("Load conf error: %s. Use default", e)
_wauto_conf = {}
_wauto_conf['priority'] = {
'update': 5,
'home_timeline': 4,
'forward': 3,
'reply': 3
}
logger.debug("conf: %s", _wauto_conf)
'''
Make the invokation from Python interpreter more convenient.
Use synchronous calls.
'''
def _dummy_decorator_generator(*args, **kwargs):
def _dummy_decorator(func):
return func
return _dummy_decorator
if __name__ == '__main__':
rate_limit = _dummy_decorator_generator
def cal_bucket(upperbound, period):
'''
class WeiboAutomator(object):
'''Wrap common operations with rate limit facility
'''
# Most buckets are derived from Sina's offcial description [1].
# The additional one 'wauto_snsapi' limits SNSAPI request rate globally
# (avoid lower layer failure).
#
# Ref:
# * [1] http://open.weibo.com/wiki/Rate-limiting
SINA_BUCKETS = [
('wauto_snsapi', LeakyBucket(1, 0, 0.5)),
('ip.hour.test_auth', cal_bucket(1000, 60 * 60)),
('user.hour.test_auth.total', cal_bucket(150, 60 * 60)),
('user.hour.test_auth.update', cal_bucket(30, 60 * 60)),
('user.hour.test_auth.reply', cal_bucket(60, 60 * 60)),
('user.hour.test_auth.follow', cal_bucket(60, 60 * 60)),
('user.day.test_auth.follow', cal_bucket(100, 60 * 60 * 24)),
]
POLICY_GROUP = {}
POLICY_GROUP['general'] = {
'wauto_snsapi': 1,
'ip.hour.test_auth': 1,
'user.hour.test_auth.total': 1
}
POLICY_GROUP['update'] = dict(POLICY_GROUP['general'],
**{'user.hour.test_auth.update': 1})
POLICY_GROUP['reply'] = dict(POLICY_GROUP['general'],
**{'user.hour.test_auth.reply': 1})
POLICY_GROUP['follow'] = dict(
POLICY_GROUP['general'], **{
'user.hour.test_auth.follow': 1,
'user.day.test_auth.follow': 1
})
_log = lambda x: logger.debug('ret: %s', x)
def __init__(self):
super(WeiboAutomator, self).__init__()
self.sp = SNSPocket()
self.sp.load_config()
self.sp.auth()
# assign 'channel_name' as automator
self.weibo = self.sp['automator']
self.rlq = RateLimitQueue()
map(lambda t: self.rlq.add_bucket(t[0], t[1]), self.SINA_BUCKETS)
# This implementation of dumps and loads are too simple.
# They do not work with some callback functions.
# I switch to 'dill'
#def dumps(self):
# r = copy.deepcopy(self.rlq)
# for t in r._tasks:
# # First arg should be 'self' if do not operate our RLQ directly.
# t.args = list(t.args)
# t.args.pop(0)
# t.func = marshal.dumps(t.func.func_code)
# t.callback = marshal.dumps(t.callback.func_code)
# return pickle.dumps(r)
#def loads(self, s):
# r = pickle.loads(s)
# for t in r._tasks:
# t.args.insert(0, self)
# t.args = tuple(t.args)
# code_func = marshal.loads(t.func)
# t.func = types.FunctionType(code_func, globals())
# code_callback = marshal.loads(t.callback)
# t.callback = types.FunctionType(code_callback, globals())
# self.rlq = r
#Original 'loads' 2
# #self.rlq._buckets = r._buckets
# #for t in r._tasks:
# # code = marshal.loads(t.callback)
# # t.callback = types.FunctionType(code, globals())
# # t.args.insert(0, self)
# # t.args = tuple(t.args)
# # t.kwargs['callback'] = t.callback
# # f = getattr(WeiboAutomator, t.func)
# # # Execute the wrapped class method again to insert task
# # f(*t.args, **t.kwargs)
def run(self):
return self.rlq.run()
def clear_tasks(self):
return self.rlq.clear_tasks()
def _tounicode(self, text):
if isinstance(text, unicode):
return text
else:
return text.decode('utf-8')
def get_uid(self):
if hasattr(self, '_uid'):
return self._uid
else:
ret = self.weibo.weibo_request('account/get_uid', 'GET', {})
return ret['uid']
uid = property(get_uid)
def rate_limit_status(self):
ret = self.weibo.weibo_request('account/rate_limit_status', 'GET', {})
return ret
@rate_limit(buckets=POLICY_GROUP['follow'], callback=_log)
def follow(self, uid):
ret = self.weibo.weibo_request('friendships/create', 'POST',
{'uid': uid})
return ret
@rate_limit(buckets=POLICY_GROUP['follow'], callback=_log)
def follow_by_name(self, screen_name):
ret = self.weibo.weibo_request('friendships/create', 'POST',
{'screen_name': screen_name})
return ret
@rate_limit(buckets=POLICY_GROUP['general'],
callback=_log,
priority=_wauto_conf['priority']['home_timeline'])
def home_timeline(self, count=20):
return self.weibo.home_timeline(count)
@rate_limit(buckets=POLICY_GROUP['update'],
callback=_log,
priority=_wauto_conf['priority']['update'])
def update(self, text):
return self.weibo.update(self._tounicode(text))
@rate_limit(buckets=POLICY_GROUP['reply'],
callback=_log,
priority=_wauto_conf['priority']['reply'])
def reply(self, status, text):
if isinstance(status, snsapi.snstype.Message):
statusID = status.ID
else:
statusID = status
return self.weibo.reply(statusID, self._tounicode(text))
@rate_limit(buckets=POLICY_GROUP['general'],
callback=_log,
priority=_wauto_conf['priority']['forward'])
def forward(self, status, text):
return self.weibo.forward(status, self._tounicode(text))
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def show(self, uid=None, screen_name=None):
params = {}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
ret = self.weibo.weibo_request('users/show', 'GET', params)
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def domain_show(self, url):
'''Lookup user by personal url.
We will match and remove common weibo prefix.
:param url:
e.g. 'http://weibo.com/xiena' --> url='xiena'
'''
import re
pattern = re.compile('^http:\/\/.*weibo.com\/')
url = re.sub(pattern, '', url)
ret = self.weibo.weibo_request('users/domain_show', 'GET',
{'domain': url})
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def get_friends(self, uid=None, screen_name=None, count=200, cursor=None):
params = {'count': count}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
if not cursor is None:
params['cursor'] = cursor
ret = self.weibo.weibo_request('friendships/friends', 'GET', params)
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def get_friends_ids(self,
uid=None,
screen_name=None,
count=5000,
cursor=None):
params = {'count': count}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
if not cursor is None:
params['cursor'] = cursor
ret = self.weibo.weibo_request('friendships/friends/ids', 'GET',
params)
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def get_followers(self,
uid=None,
screen_name=None,
count=200,
cursor=None):
params = {'count': count}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
if not cursor is None:
params['cursor'] = cursor
ret = self.weibo.weibo_request('friendships/followers', 'GET', params)
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def get_followers_ids(self,
uid=None,
screen_name=None,
count=5000,
cursor=None):
params = {'count': count}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
if not cursor is None:
params['cursor'] = cursor
ret = self.weibo.weibo_request('friendships/followers/ids', 'GET',
params)
return ret
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def get_followers_active(self,
uid=None,
screen_name=None,
count=200,
cursor=None):
params = {'count': count}
if not uid is None:
params['uid'] = uid
elif not screen_name is None:
params['screen_name'] = screen_name
else:
params['uid'] = self.uid
if not cursor is None:
params['cursor'] = cursor
ret = self.weibo.weibo_request('friendships/followers/active', 'GET',
params)
return ret
# This Api is only for advanced app permission
@rate_limit(buckets=POLICY_GROUP['general'], callback=_log)
def search_topics(self, q, count=50, page=None):
params = {'count': count}
if not page is None:
params['page'] = page
ret = self.weibo.weibo_request('search/topics', 'GET', params)
return ret
def run(self):
while (self.keep_running):
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(60 * 5) # 5 Minutes per fetch
q = SRFEQueue(sp)
q.connect()
q.refresh_tags()
try:
jsonconf = json.load(open('conf/srfe.json', 'r'))
except IOError:
logger.warning("Do not find conf/srfe.json, use defaults")
jsonconf = {
"cookie_sign_key": "You had better set your own key in the config!!!!",
"username": "******",
"password": "******"
}
INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch
logger.debug("INPUT_GAP: %s", INPUT_GAP)
class InputThread(threading.Thread):
def __init__(self, queue):
super(InputThread, self).__init__()
self.queue = queue
self.keep_running = True
def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def _inqueue(self, message):
cur = self.con.cursor()
try:
# Deduplicate
# Explain the problem of the following two methods for future reference:
# 1. digest = self._digest_pyobj(message)
# Python object are hashed to different values even the SNS message
# fields are all the same.
# 2. digest = message.digest_parsed()
# I forget what is the problem.. I should have noted before.
digest = message.digest()
#logger.debug("message pyobj digest '%s'", digest)
r = cur.execute(
'''
SELECT digest FROM msg
WHERE digest = ?
''', (digest, ))
if len(list(r)) > 0:
#logger.debug("message '%s' already exists", digest)
return False
else:
logger.debug("message '%s' is new", digest)
#TODO:
# This is temporary solution for object digestion.
#
# For our Message object, the following evaluates to False!!
# Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o)))
#
# To perform deduplication and further refer to this message,
# we store the calculated digestion as an attribute of the message.
# Note however, after this operation the digest of 'message' will not
# be the valued stored therein! This is common problem in such mechanism,
# e.g. UDP checksum. Developers should have this in mind.
message.digest_pyobj = self._digest_pyobj(message)
cur.execute('''
INSERT INTO msg(
time ,
text ,
userid ,
username ,
mid ,
platform ,
digest ,
digest_parsed ,
digest_pyobj ,
parsed ,
pyobj ,
flag ,
weight ,
weight_time
)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
''', (\
message.parsed.time,\
message.parsed.text,\
message.parsed.userid,\
message.parsed.username,\
str(message.ID),\
message.platform,\
message.digest(),\
message.digest_parsed(),\
#self._digest_pyobj(message),\
message.digest_pyobj,\
message.dump_parsed(),\
self._pyobj2str(message),\
"unseen",
self._weight_feature(message),
int(time.time())
))
return True
except Exception, e:
logger.warning("failed: %s", str(e))
#print message
#raise e
return False
q = SRFEQueue(sp)
q.connect()
q.refresh_tags()
try:
jsonconf = json.load(open('conf/srfe.json', 'r'))
except IOError:
logger.warning("Do not find conf/srfe.json, use defaults")
jsonconf = {
"cookie_sign_key": "You had better set your own key in the config!!!!",
"username": "******",
"password": "******"
}
INPUT_GAP = jsonconf.get('input_gap', 60 * 5) # 5 Minutes per fetch
logger.debug("INPUT_GAP: %s", INPUT_GAP)
class InputThread(threading.Thread):
def __init__(self, queue):
super(InputThread, self).__init__()
self.queue = queue
self.keep_running = True
def run(self):
while (self.keep_running):
try:
self.queue.input()
logger.debug("Invoke input() on queue")
time.sleep(INPUT_GAP)
except Exception as e:
logger.warning("Catch Exception in InputThread: %s", e)
def _inqueue(self, message):
cur = self.con.cursor()
try:
# Deduplicate
# Explain the problem of the following two methods for future reference:
# 1. digest = self._digest_pyobj(message)
# Python object are hashed to different values even the SNS message
# fields are all the same.
# 2. digest = message.digest_parsed()
# I forget what is the problem.. I should have noted before.
digest = message.digest()
#logger.debug("message pyobj digest '%s'", digest)
r = cur.execute('''
SELECT digest FROM msg
WHERE digest = ?
''', (digest, ))
if len(list(r)) > 0:
#logger.debug("message '%s' already exists", digest)
return False
else:
logger.debug("message '%s' is new", digest)
self._hook_new_message(self, message)
#TODO:
# This is temporary solution for object digestion.
#
# For our Message object, the following evaluates to False!!
# Serialize.dumps(o) == Serialize.dumps(Serialize.loads(Serialize.dumps(o)))
#
# To perform deduplication and further refer to this message,
# we store the calculated digestion as an attribute of the message.
# Note however, after this operation the digest of 'message' will not
# be the valued stored therein! This is common problem in such mechanism,
# e.g. UDP checksum. Developers should have this in mind.
message.digest_pyobj = self._digest_pyobj(message)
cur.execute('''
INSERT INTO msg(
time ,
text ,
userid ,
username ,
mid ,
platform ,
digest ,
digest_parsed ,
digest_pyobj ,
parsed ,
pyobj ,
flag ,
weight ,
weight_time
)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
''', (\
message.parsed.time,\
message.parsed.text,\
message.parsed.userid,\
message.parsed.username,\
str(message.ID),\
message.platform,\
message.digest(),\
message.digest_parsed(),\
#self._digest_pyobj(message),\
message.digest_pyobj,\
message.dump_parsed(),\
self._pyobj2str(message),\
"unseen",
self._weight_feature(message),
int(self.time())
))
return True
except Exception, e:
logger.warning("failed: %s", str(e))
#print message
#raise e
return False
