def getNewWechatExtInfo():
'''
modify by bigzhu at 15/09/13 17:25:43 为了用订阅号给指定用户发消息，改用WechatExt
'''
wechat_ext = WechatExt(user_name, password,
appid=appid
)
token_cookies = wechat_ext.get_token_cookies()
pg.insertOrUpdate(pg, 'wechat_dead_line', token_cookies, where='1=1')
return wechat_ext
def getNewWechatExtInfo():
'''
modify by bigzhu at 15/09/13 17:25:43 为了用订阅号给指定用户发消息，改用WechatExt
'''
wechat_ext = WechatExt(user_name, password,
appid=appid
)
token_cookies = wechat_ext.get_token_cookies()
pg.insertOrUpdate(pg, 'wechat_dead_line', token_cookies, where='1=1')
return wechat_ext
def saveUserCheckNew(blogs):
'''
create by bigzhu at 15/09/05 11:57:11
'''
user = blogs['blog']
user_name = user['name']
where = "name='%s'" % user_name
result = list(pg.select('tumblr_user', where=where))
if result:
if result[0].updated == user['updated']: # 如果没有更新过，就不用继续了
print user_name, ' no update'
# return
pass
pg.insertOrUpdate(pg, 'tumblr_user', user, where)
blogs = blogs['posts']
saveBlogs(user_name, blogs, offset=20)
def getUser(user_name, always_check=False):
'''
modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
'''
users = list(
pg.select('instagram_user',
where="lower(username)=lower('%s')" % user_name))
if users and not always_check:
return users[0]
else:
user = None
try:
for this_user in api.user_search(user_name, 10):
if this_user.username.lower() == user_name.lower():
user = this_user
break
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
return
# 如果没有这个用户
if user is None:
public_db.delNoName('instagram', user_name)
return
try:
user = api.user(user.id)
except (instagram.bind.InstagramAPIError,
instagram.bind.InstagramClientError):
# 通常是没有访问权限
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
db_user = storage()
db_user.id_str = user.id
db_user.username = user.username
db_user.full_name = user.full_name
db_user.profile_picture = user.profile_picture
db_user.bio = user.bio
db_user.website = user.website
db_user.counts = json.dumps(user.counts)
#pg.insert('instagram_user', **db_user)
pg.insertOrUpdate(pg, 'instagram_user', db_user,
"id_str='%s'" % db_user.id_str)
#db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
return getUser(user_name)
def getUser(user_name, always_check=False):
'''
modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
'''
users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name))
if users and not always_check:
return users[0]
else:
user = None
try:
for this_user in api.user_search(user_name, 10):
if this_user.username.lower() == user_name.lower():
user = this_user
break
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
return
# 如果没有这个用户
if user is None:
public_db.delNoName('instagram', user_name)
return
try:
user = api.user(user.id)
except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError):
# 通常是没有访问权限
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
db_user = storage()
db_user.id_str = user.id
db_user.username = user.username
db_user.full_name = user.full_name
db_user.profile_picture = user.profile_picture
db_user.bio = user.bio
db_user.website = user.website
db_user.counts = json.dumps(user.counts)
#pg.insert('instagram_user', **db_user)
pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str)
#db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
return getUser(user_name)
def saveTwitter(tweet):
'''
create by bigzhu at 15/07/10 14:39:48
保存twitter
'''
del tweet.user._json
#del tweet.user._api
tweet.user.entities = json.dumps(tweet.user.entities)
del tweet.user.id
pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str)
tweet.t_user_id = tweet.user.id_str
screen_name = tweet.user.screen_name
del tweet.user
if hasattr(tweet, 'author'):
#del tweet.author.id
pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str)
tweet.t_author_id = tweet.author.id_str
del tweet.author
if hasattr(tweet, '_api'):
del tweet._api
if hasattr(tweet, '_json'):
del tweet._json
# twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
if hasattr(tweet, 'id'):
del tweet.id
if hasattr(tweet, 'entities'):
tweet.entities = json.dumps(tweet.entities)
if hasattr(tweet, 'geo'):
tweet.geo = json.dumps(tweet.geo)
if hasattr(tweet, 'coordinates'):
tweet.coordinates = json.dumps(tweet.coordinates)
if hasattr(tweet, 'extended_entities'):
tweet.extended_entities = json.dumps(tweet.extended_entities)
if hasattr(tweet, 'scopes'):
tweet.scopes = json.dumps(tweet.scopes)
if hasattr(tweet, 'retweeted_status'):
saveTwitter(tweet.retweeted_status)
tweet.retweeted_status = tweet.retweeted_status.id_str
if hasattr(tweet, 'quoted_status'):
# print tweet.quoted_status
del tweet.quoted_status
# saveTwitter(tweet.quoted_status)
# tweet.quoted_status = tweet.quoted_status.id_str
# place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
if hasattr(tweet, 'place'):
del tweet.place
# for k, v in vars(tweet).items():
# print '%s=%s' % (k, v)
#return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)
m = public_bz.storage()
m.id_str = tweet.id_str
m.m_type = 'twitter'
m.m_user_id = tweet.t_user_id
m.created_at = tweet.created_at
if hasattr(tweet, 'extended_entities'):
m.extended_entities = tweet.extended_entities
m.content = None
m.text = tweet.text
m.href = 'https://twitter.com/'+screen_name+'/status/'+tweet.id_str
return pg.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)
def saveTwitter(tweet):
'''
create by bigzhu at 15/07/10 14:39:48
保存twitter
'''
del tweet.user._json
#del tweet.user._api
tweet.user.entities = json.dumps(tweet.user.entities)
del tweet.user.id
pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user),
"id_str='%s'" % tweet.user.id_str)
tweet.t_user_id = tweet.user.id_str
screen_name = tweet.user.screen_name
del tweet.user
if hasattr(tweet, 'author'):
#del tweet.author.id
pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author),
"id_str='%s'" % tweet.author.id_str)
tweet.t_author_id = tweet.author.id_str
del tweet.author
if hasattr(tweet, '_api'):
del tweet._api
if hasattr(tweet, '_json'):
del tweet._json
# twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
if hasattr(tweet, 'id'):
del tweet.id
if hasattr(tweet, 'entities'):
tweet.entities = json.dumps(tweet.entities)
if hasattr(tweet, 'geo'):
tweet.geo = json.dumps(tweet.geo)
if hasattr(tweet, 'coordinates'):
tweet.coordinates = json.dumps(tweet.coordinates)
if hasattr(tweet, 'extended_entities'):
tweet.extended_entities = json.dumps(tweet.extended_entities)
if hasattr(tweet, 'scopes'):
tweet.scopes = json.dumps(tweet.scopes)
if hasattr(tweet, 'retweeted_status'):
saveTwitter(tweet.retweeted_status)
tweet.retweeted_status = tweet.retweeted_status.id_str
if hasattr(tweet, 'quoted_status'):
# print tweet.quoted_status
del tweet.quoted_status
# saveTwitter(tweet.quoted_status)
# tweet.quoted_status = tweet.quoted_status.id_str
# place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
if hasattr(tweet, 'place'):
del tweet.place
# for k, v in vars(tweet).items():
# print '%s=%s' % (k, v)
#return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)
m = public_bz.storage()
m.id_str = tweet.id_str
m.m_type = 'twitter'
m.m_user_id = tweet.t_user_id
m.created_at = tweet.created_at
if hasattr(tweet, 'extended_entities'):
m.extended_entities = tweet.extended_entities
m.content = None
m.text = tweet.text
m.href = 'https://twitter.com/' + screen_name + '/status/' + tweet.id_str
return pg.insertIfNotExist(
pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)