def query(self):
ret, data = self._wget()
logger.info('request %s - "%s"' % (ret, data))
if ret:
self._price = self._parse(data)
logger.info("price %0.2f" % (self._price))
return ret, self._price
def main():
if len(sys.argv) != 3:
logger.info('argument num is invalid - ' + str(len(sys.argv)))
logger.info(' ')
logger.info('>> python renren_crawler.py user password')
sys.exit(1)
logger.info('ren ren crawler, start...')
user = sys.argv[1]
password = sys.argv[2]
# 实例化人人蜘蛛
renren = RenRenCrawler()
# 设置cookie启用
renren.set_cookie()
# 登陆人人
renren.login(user, password)
## 获取好友列表
renren.get_friends_list()
# 获取推荐好友列表
renren.get_maybe_friends_list()
logger.info('ren ren crawler, stop.')
pass
def _store_db(self, price):
logger.info('store price - %0.2f' % (price))
sql = 'INSERT INTO t_price_history (price, updatetime) VALUES (%0.2f, %d);' % (price, int(time.time()))
ret, rec = self._db.execute(sql)
if not ret:
logger.error('store price error - %0.2f' % (price))
pass
def query_self(self):
ret, data = self._wget()
logger.info('request %s - "%s"' % (ret, data))
#if ret:
# self._price = self._parse(data)
# logger.info('price %0.2f' % (self._price))
return data
def query(self):
ret, data = self._wget()
logger.info('request %s - "%s"' % (ret, data))
if ret:
self._price = self._parse(data)
logger.info('price %0.2f' % (self._price))
return ret, self._price
def main():
if len(sys.argv) != 3:
logger.info('argument num is invalid - ' + str(len(sys.argv)))
logger.info(' ')
logger.info('>> python renren_crawler.py user password')
sys.exit(1)
logger.info('ren ren crawler, start...')
user = sys.argv[1]
password = sys.argv[2]
# 实例化人人蜘蛛
renren = RenRenCrawler()
# 设置cookie启用
renren.set_cookie()
# 登陆人人
renren.login(user, password)
## 获取好友列表
renren.get_friends_list()
# 获取推荐好友列表
renren.get_maybe_friends_list()
logger.info('ren ren crawler, stop.')
pass
def _store_db(self, price):
logger.info('store price - %0.2f' % (price))
sql = 'INSERT INTO t_price_history (price, updatetime) VALUES (%0.2f, %d);' % (
price, int(time.time()))
ret, rec = self._db.execute(sql)
if not ret:
logger.error('store price error - %0.2f' % (price))
pass
def __init__(self): #, influxdb_client):
self.fxrio = Fixerio()
self.exchange_data = {}
self._name = 'Exchange'
self.baselist = {'CNY', 'USD', 'KRW'}
self.exchange_data = {}
for index in self.baselist:
self.exchange_data[index] = None
logger.info('Exchange module init')
def _sync(self):
for cnt in range(1, self._retry+1):
logger.info('%s sync account - %d' % (self._TITLE, cnt))
if self._do_sync():
logger.info('%s sync account - ok (btc %s, cny %s)' % (self._TITLE, self._balance['btc']['amount'], self._balance['cny']['amount']))
break
time.sleep(self._retry_interval)
pass
else:
logger.error('%s sync account - fail' % (self._TITLE))
def run(self):
while True:
try:
self.update_btcchina()
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.info('sleep %d sec --------------------------------------' % (config.PRICE_UPDATE_INTERVAL))
time.sleep(config.PRICE_UPDATE_INTERVAL)
pass
def run(self):
while True:
try:
self.update_btcchina()
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.info('sleep %d sec --------------------------------------' %
(config.PRICE_UPDATE_INTERVAL))
time.sleep(config.PRICE_UPDATE_INTERVAL)
pass
def _sync(self):
for cnt in range(1, self._retry + 1):
logger.info('%s sync account - %d' % (self._TITLE, cnt))
if self._do_sync():
logger.info('%s sync account - ok (btc %s, cny %s)' %
(self._TITLE, self._balance['btc']['amount'],
self._balance['cny']['amount']))
break
time.sleep(self._retry_interval)
pass
else:
logger.error('%s sync account - fail' % (self._TITLE))
def _fresh_pipeline(self, price_tuple_history):
now = time.time()
fresh_price_history = []
for tuple_price in price_tuple_history:
price = tuple_price[0]
updatetime = tuple_price[1]
elapse_sec = now - updatetime
if elapse_sec < (config.STOP_LOSS_FRESH_TIME * 60):
fresh_price_history.append(price)
else:
logger.info('price info too old - %d secs - "%s"' % (elapse_sec, str(tuple_price)))
logger.info('fresh pipeline %d -> %d' % (len(price_tuple_history), len(fresh_price_history)))
return fresh_price_history
def _fresh_pipeline(self, price_tuple_history):
now = time.time()
fresh_price_history = []
for tuple_price in price_tuple_history:
price = tuple_price[0]
updatetime = tuple_price[1]
elapse_sec = now - updatetime
if elapse_sec < (config.STOP_LOSS_FRESH_TIME * 60):
fresh_price_history.append(price)
else:
logger.info('price info too old - %d secs - "%s"' %
(elapse_sec, str(tuple_price)))
logger.info('fresh pipeline %d -> %d' %
(len(price_tuple_history), len(fresh_price_history)))
return fresh_price_history
def __init__(self):
self._agents = []
self.exchange_data = {}
self.exchange_client = Exchange()
ex_td = threading.Thread(target=self.update_exchange)
self.isRun = True
ex_td.start()
logger.info('Exchange updater start')
self.idb = InfluxDBHelper()
self.pum = PriceUpdater(self.idb)
self._load_agent()
pass
class Sqlite3(object):
'''
'''
def __init__(self):
self._db_file = config.DB_PATH
self._sql_create_tables = config.SQL_CREATE_TABLES
self._create_db()
pass
def _create_db(self):
ret = False
try:
conn = sqlite3.connect(self._db_file)
logger.info('db file - "%s"' % (self._db_file))
c = conn.cursor()
for k in self._sql_create_tables:
c.execute(self._sql_create_tables[k])
conn.commit()
logger.info('create table "%s"' % (k))
conn.close()
ret = True
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.error('create table fail')
return ret
logger.info('create table ok')
return ret
def _create_db(self):
ret = False
try:
conn = sqlite3.connect(self._db_file)
logger.info('db file - "%s"' % (self._db_file))
c = conn.cursor()
for k in self._sql_create_tables:
c.execute(self._sql_create_tables[k])
conn.commit()
logger.info('create table "%s"' % (k))
conn.close()
ret = True
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.error('create table fail')
return ret
def _do_sell(self, price, amount):
try:
# set user id
result = self._bc.get_account_info(post_data={'id': self._id})
# TODO FOR TEST
#raise NameError
#result = True
result = self._bc.sell(price, amount)
logger.info('sell call - "%s"' % (str(result)))
if not result:
return False
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
return False
def _create_db(self):
ret = False
try:
conn = sqlite3.connect(self._db_file)
logger.info('db file - "%s"' % (self._db_file))
c = conn.cursor()
for k in self._sql_create_tables:
c.execute(self._sql_create_tables[k])
conn.commit()
logger.info('create table "%s"' % (k))
conn.close()
ret = True
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.error('create table fail')
return ret
def _do_sell(self, price, amount):
try:
# set user id
result = self._bc.get_account_info(post_data={'id':self._id})
# TODO FOR TEST
#raise NameError
#result = True
result = self._bc.sell(price, amount)
logger.info('sell call - "%s"' % (str(result)))
if not result:
return False
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
return False
def run(self):
while True:
logger.info('sleep %d sec' % (config.PRICE_MONITOR_INTERVAL))
time.sleep(config.PRICE_MONITOR_INTERVAL)
ret, recs = self._load_recent_price()
if ret:
# 判断价格历史数据是否已经过期
price_history = self._fresh_pipeline(recs)
if len(price_history) < (config.STOP_LOSS_SAMPLE_SPACE / 10):
logger.info('stop loss sample space is too small %d/%d, skip' % (len(price_history), config.STOP_LOSS_SAMPLE_SPACE))
continue
# 止损策略
self._strategy_stop_loss_mean(price_history)
pass
pass
pass
def execute(self, sql):
ret = False
rec_list = []
try:
conn = sqlite3.connect(self._db_file)
c = conn.cursor()
c.execute(sql)
conn.commit()
logger.info('execute - "%s"' % (sql))
rec_list = c.fetchall()
logger.info('retrive - %d records - "%s"' % (len(rec_list), str(rec_list)))
conn.close()
ret = True
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.error('execute sql fail - "%s"' % (sql))
return ret, rec_list
def execute(self, sql):
ret = False
rec_list = []
try:
conn = sqlite3.connect(self._db_file)
c = conn.cursor()
c.execute(sql)
conn.commit()
logger.info('execute - "%s"' % (sql))
rec_list = c.fetchall()
logger.info('retrive - %d records - "%s"' %
(len(rec_list), str(rec_list)))
conn.close()
ret = True
except Exception, e:
logger.error(traceback.format_exc())
logger.error(str(e))
logger.error('execute sql fail - "%s"' % (sql))
return ret, rec_list
def run(self):
while True:
logger.info('sleep %d sec' % (config.PRICE_MONITOR_INTERVAL))
time.sleep(config.PRICE_MONITOR_INTERVAL)
ret, recs = self._load_recent_price()
if ret:
# 判断价格历史数据是否已经过期
price_history = self._fresh_pipeline(recs)
if len(price_history) < (config.STOP_LOSS_SAMPLE_SPACE / 10):
logger.info(
'stop loss sample space is too small %d/%d, skip' %
(len(price_history), config.STOP_LOSS_SAMPLE_SPACE))
continue
# 止损策略
self._strategy_stop_loss_mean(price_history)
pass
pass
pass
def query_history(self, history_length, file_path):
price_list = []
ret, data = self.execute('select price from t_price_history order by updatetime desc limit %d' % (history_length))
if not ret:
logger.error('query history fail')
return ret
for item in data:
price_list.append(str(int(item[0])))
price_list.reverse()
fp = open(file_path, 'w')
fp.write('\r\n'.join(price_list))
fp.close()
logger.info('query history dump latest %d price data to file "%s"' % (history_length, file_path))
return ret
pass
def login(self, user, password):
headers = {
'Host': 'www.renren.com',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-cn,zh;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Referer': 'http://www.renren.com/SysHome.do',
#'Content-Length': '133', # USELESS
#'Cookie': '_r01_=1; depovince=BJ; jebecookies=f6bce625-6dbc-4f9a-9a29-58a629eac5da|||||; idc=tel; ick=35776a1f-d2e0-4f25-805a-59533875f319; loginfrom=null; feedType=37491_hot; JSESSIONID=abcgo0AVJR7s7fCIqDdBt; anonymid=h15tajwhbuns1e', # USELESS
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
param = {
'email': user,
'password': password,
'icode': '',
'origURL': 'http://www.renren.com/home',
'domain': 'renren.com',
'key_id': '1',
'_rtk': 'c59b27d3',
}
url = 'http://renren.com/ajaxLogin/login'
postdata = urllib.urlencode(param)
req = urllib2.Request(url, postdata, headers)
login_response = urllib2.urlopen(req)
logger.info('login ' + user)
ret = login_response.read()
#logger.info(ret)
pass
def login(self, user, password):
headers = {
'Host': 'www.renren.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-cn,zh;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Referer': 'http://www.renren.com/SysHome.do',
#'Content-Length': '133', # USELESS
#'Cookie': '_r01_=1; depovince=BJ; jebecookies=f6bce625-6dbc-4f9a-9a29-58a629eac5da|||||; idc=tel; ick=35776a1f-d2e0-4f25-805a-59533875f319; loginfrom=null; feedType=37491_hot; JSESSIONID=abcgo0AVJR7s7fCIqDdBt; anonymid=h15tajwhbuns1e', # USELESS
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
param = {
'email': user,
'password': password,
'icode': '',
'origURL': 'http://www.renren.com/home',
'domain': 'renren.com',
'key_id': '1',
'_rtk': 'c59b27d3',
}
url = 'http://renren.com/ajaxLogin/login'
postdata=urllib.urlencode(param)
req = urllib2.Request(url, postdata, headers)
login_response= urllib2.urlopen(req)
logger.info('login ' + user)
ret =login_response.read()
#logger.info(ret)
pass
def query_history(self, history_length, file_path):
price_list = []
ret, data = self.execute(
'select price from t_price_history order by updatetime desc limit %d'
% (history_length))
if not ret:
logger.error('query history fail')
return ret
for item in data:
price_list.append(str(int(item[0])))
price_list.reverse()
fp = open(file_path, 'w')
fp.write('\r\n'.join(price_list))
fp.close()
logger.info('query history dump latest %d price data to file "%s"' %
(history_length, file_path))
return ret
pass
def update_btcchina(self):
#raise NameError('Exception Raise')
logger.info('task start - agents total %d' % (len(self._agents)))
for agent in self._agents:
logger.info('agent "%s"' % (agent.name))
# get price
logger.info('1. get price')
ret, price = agent.query()
if not ret:
logger.error('query failed, skip "%s"' % (agent.name))
continue
# store
logger.info('2. store db')
self._store_db(price)
pass
def update_btcchina(self):
#raise NameError('Exception Raise')
logger.info('task start - agents total %d' % (len(self._agents)))
for agent in self._agents:
logger.info('agent "%s"' % (agent.name))
# get price
logger.info('1. get price')
ret, price = agent.query()
if not ret:
logger.error('query failed, skip "%s"' % (agent.name))
continue
# store
logger.info('2. store db')
self._store_db(price)
pass
def sell(self, price, amount):
''' buy and sell require price (CNY, 5 decimals) and amount (BTC, 8 decimals) '''
price = float('%0.5f' % price)
amount = float('%0.8f' % amount)
for cnt in range(1, self._retry+1):
logger.info('%s sell btc %0.5f on %0.5f CNY - %d' % (self._TITLE, amount, price, cnt))
if self._do_sell(price, amount):
logger.info('%s sell --OK--' % (self._TITLE))
break
time.sleep(self._retry_interval)
pass
else:
logger.info('%s sell --FAIL-- btc %0.5f on %0.5f CNY - %d' % (self._TITLE, price, amount, cnt))
def sell(self, price, amount):
''' buy and sell require price (CNY, 5 decimals) and amount (BTC, 8 decimals) '''
price = float('%0.5f' % price)
amount = float('%0.8f' % amount)
for cnt in range(1, self._retry + 1):
logger.info('%s sell btc %0.5f on %0.5f CNY - %d' %
(self._TITLE, amount, price, cnt))
if self._do_sell(price, amount):
logger.info('%s sell --OK--' % (self._TITLE))
break
time.sleep(self._retry_interval)
pass
else:
logger.info('%s sell --FAIL-- btc %0.5f on %0.5f CNY - %d' %
(self._TITLE, price, amount, cnt))
def sell_out(self, price):
# sync account
self._sync()
# get all btc
btc_amount = float(self._balance['btc']['amount'])
now = time.time()
if now - self._last_sell_out < config.MIN_INTERVAL_SELLOUT:
logger.info('%s sell out operation too frequently, cancel - btc %f on %f' % (self._TITLE, btc_amount, price))
return
self._last_sell_out = now
logger.info('%s sell out begin' % (self._TITLE))
self.sell(price, btc_amount)
logger.info('%s sell out end' % (self._TITLE))
pass
def send(self, msg):
ret = False
if not self._check_interval():
logger.info('sms send operation too frequently, cancel - "%s"' % msg)
return ret
url = self._gateway_url % (",".join(self._tel_list), msg)
ret, data = self._wget(url)
logger.info('request %s - "%s"' % (ret, data))
if ret:
json_data = json.loads(data)
if json_data["ret"] == "0":
logger.info('sms send ok - "%s"' % (msg))
else:
ret = False
logger.error('sms send fail - "%s"' % (msg))
return ret
def send(self, msg):
ret = False
if not self._check_interval():
logger.info('sms send operation too frequently, cancel - "%s"' %
msg)
return ret
url = self._gateway_url % (','.join(self._tel_list), msg)
ret, data = self._wget(url)
logger.info('request %s - "%s"' % (ret, data))
if ret:
json_data = json.loads(data)
if json_data['ret'] == '0':
logger.info('sms send ok - "%s"' % (msg))
else:
ret = False
logger.error('sms send fail - "%s"' % (msg))
return ret
def sell_out(self, price):
# sync account
self._sync()
# get all btc
btc_amount = float(self._balance['btc']['amount'])
now = time.time()
if now - self._last_sell_out < config.MIN_INTERVAL_SELLOUT:
logger.info(
'%s sell out operation too frequently, cancel - btc %f on %f' %
(self._TITLE, btc_amount, price))
return
self._last_sell_out = now
logger.info('%s sell out begin' % (self._TITLE))
self.sell(price, btc_amount)
logger.info('%s sell out end' % (self._TITLE))
pass
def _strategy_stop_loss_mean(self, price_history):
ret = False
msg = ''
# 计算平均值
mean = sum(price_history) / float(len(price_history))
if mean < config.STOP_LOSS_MEAN:
ret = True
last = price_history[0]
if not ret:
logger.info('>>> strategy mean checked safe - mean %0.2f' % (mean))
return ret, msg
logger.info(
'>>> BE CAUTIOUS -> strategy mean checked DANGEROUS - mean %0.2f / %0.2f'
% (mean, config.STOP_LOSS_MEAN))
# 发送预警短信
if config.STOP_LOSS_EXEC_SMS:
logger.info('exec sms flag is open')
msg = 'kxLowPrice,sms_zend_framework,sl,10min,mean%0.2f,last%0.2f' % (
mean, last)
self._sms.send(msg)
else:
logger.info('exec sms flag is close')
# 执行抛售操作
if config.STOP_LOSS_EXEC_SELLOUT:
logger.info('exec sell flag is open')
best_ask = last - config.OFFSET_SELLOUT
logger.info('best ask %0.2f' % (best_ask))
self._account_btcchina.sell_out(best_ask)
else:
logger.info('exec sms flag is close')
return ret, msg
def set_cookie(self):
# 设置cookie
cookie=cookielib.CookieJar()
opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
logger.info('set cookie')
def set_cookie(self):
# 设置cookie
cookie = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
logger.info('set cookie')
def get_friends_list(self):
friends = []
friends_id = []
req=urllib2.urlopen('http://friend.renren.com/myfriendlistx.do#item_0')
ret_html =req.read()
logger.info('get friends list')
friends_json = ''
m = re.search(r'var friends=(.*);', ret_html)
if m:
friends_json = m.group(1)
else:
logger.info('can`t find friend list')
return False, friends, friends_id
# head
# <type 'unicode'>
# name
# <type 'unicode'>
# mo
# <type 'bool'>
# selected
# <type 'bool'>
# vip
# <type 'bool'>
# groups
# <type 'list'>
# id
# <type 'int'>
friends = json.loads(friends_json)
logger.info('friends total:\t' + str(len(friends)))
for f in friends:
friends_id.append(f['id'])
logger.info('name: \t' + f['name'])
logger.info('id: \t' + str(f['id']))
logger.info('mo: \t' + str(f['mo']))
logger.info('vip: \t' + str(f['vip']))
logger.info('selected:\t' + str(f['selected']))
logger.info('groups: \t')
for g in f['groups']:
logger.info(' \t' + g)
logger.info('')
logger.info('friends id: ' + str(friends_id))
return True, friends, friends_id
def _strategy_stop_loss_mean(self, price_history):
ret = False
msg = ''
# 计算平均值
mean = sum(price_history) / float(len(price_history))
if mean < config.STOP_LOSS_MEAN:
ret = True
last = price_history[0]
if not ret:
logger.info('>>> strategy mean checked safe - mean %0.2f' % (mean))
return ret, msg
logger.info('>>> BE CAUTIOUS -> strategy mean checked DANGEROUS - mean %0.2f / %0.2f' % (mean, config.STOP_LOSS_MEAN))
# 发送预警短信
if config.STOP_LOSS_EXEC_SMS:
logger.info('exec sms flag is open')
msg = 'kxLowPrice,sms_zend_framework,sl,10min,mean%0.2f,last%0.2f' % (mean, last)
self._sms.send(msg)
else:
logger.info('exec sms flag is close')
# 执行抛售操作
if config.STOP_LOSS_EXEC_SELLOUT:
logger.info('exec sell flag is open')
best_ask = last - config.OFFSET_SELLOUT
logger.info('best ask %0.2f' % (best_ask))
self._account_btcchina.sell_out(best_ask)
else:
logger.info('exec sms flag is close')
return ret, msg
def exchange_callback(self, dic):
self.exchange_data = dic
self.pum.set_exchange(self.exchange_data)
logger.info('exchange callback : %s' % self.exchange_data)
def get_friends_list(self):
friends = []
friends_id = []
req = urllib2.urlopen(
'http://friend.renren.com/myfriendlistx.do#item_0')
ret_html = req.read()
logger.info('get friends list')
friends_json = ''
m = re.search(r'var friends=(.*);', ret_html)
if m:
friends_json = m.group(1)
else:
logger.info('can`t find friend list')
return False, friends, friends_id
# head
# <type 'unicode'>
# name
# <type 'unicode'>
# mo
# <type 'bool'>
# selected
# <type 'bool'>
# vip
# <type 'bool'>
# groups
# <type 'list'>
# id
# <type 'int'>
friends = json.loads(friends_json)
logger.info('friends total:\t' + str(len(friends)))
for f in friends:
friends_id.append(f['id'])
logger.info('name: \t' + f['name'])
logger.info('id: \t' + str(f['id']))
logger.info('mo: \t' + str(f['mo']))
logger.info('vip: \t' + str(f['vip']))
logger.info('selected:\t' + str(f['selected']))
logger.info('groups: \t')
for g in f['groups']:
logger.info(' \t' + g)
logger.info('')
logger.info('friends id: ' + str(friends_id))
return True, friends, friends_id
def get_maybe_friends_list(self):
maybe_friends = []
maybe_friends_id = []
req=urllib2.urlopen('http://friend.renren.com/myrelationperson.do')
ret_html =req.read()
logger.info('get maybe friends list')
maybe_friends_json = ''
m = re.search(r'var maybe_groups = (.*);', ret_html)
if m:
maybe_friends_json = m.group(1)
else:
logger.info('can`t find maybe friend list')
return False, maybe_friends, maybe_friends_id
maybe_friends = json.loads(maybe_friends_json)
logger.info('maybe friends total:\t' + str(len(maybe_friends)))
for fs in maybe_friends:
logger.info('name: \t' + fs['name'])
logger.info('type: \t' + fs['type'])
logger.info('morelink: \t' + fs['morelink'])
logger.info('count: \t' + str(fs['count']))
for f in fs['people']:
maybe_friends_id.append(f['id'])
indent = ' '*4
logger.info(indent + 'name: \t' + f['name'])
logger.info(indent + 'id: \t' + str(f['id']))
logger.info(indent + 'gender: \t' + f['gender'])
logger.info(indent + 'tinyurl: \t' + f['tinyurl'])
logger.info('')
logger.info('maybe friends id: ' + str(maybe_friends_id))
return True, maybe_friends, maybe_friends_id
def get_maybe_friends_list(self):
maybe_friends = []
maybe_friends_id = []
req = urllib2.urlopen('http://friend.renren.com/myrelationperson.do')
ret_html = req.read()
logger.info('get maybe friends list')
maybe_friends_json = ''
m = re.search(r'var maybe_groups = (.*);', ret_html)
if m:
maybe_friends_json = m.group(1)
else:
logger.info('can`t find maybe friend list')
return False, maybe_friends, maybe_friends_id
maybe_friends = json.loads(maybe_friends_json)
logger.info('maybe friends total:\t' + str(len(maybe_friends)))
for fs in maybe_friends:
logger.info('name: \t' + fs['name'])
logger.info('type: \t' + fs['type'])
logger.info('morelink: \t' + fs['morelink'])
logger.info('count: \t' + str(fs['count']))
for f in fs['people']:
maybe_friends_id.append(f['id'])
indent = ' ' * 4
logger.info(indent + 'name: \t' + f['name'])
logger.info(indent + 'id: \t' + str(f['id']))
logger.info(indent + 'gender: \t' + f['gender'])
logger.info(indent + 'tinyurl: \t' + f['tinyurl'])
logger.info('')
logger.info('maybe friends id: ' + str(maybe_friends_id))
return True, maybe_friends, maybe_friends_id