def saveTwitter(tweet):
'''
create by bigzhu at 15/07/10 14:39:48
保存twitter
'''
if hasattr(tweet, 'user'):
del tweet.user._json
#del tweet.user._api
tweet.user.entities = json.dumps(tweet.user.entities)
del tweet.user.id
db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str)
tweet.t_user_id = tweet.user.id_str
del tweet.user
if hasattr(tweet, 'author'):
#del tweet.author.id
db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str)
tweet.t_author_id = tweet.author.id_str
del tweet.author
if hasattr(tweet, '_api'):
del tweet._api
if hasattr(tweet, '_json'):
del tweet._json
# twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
if hasattr(tweet, 'id'):
del tweet.id
if hasattr(tweet, 'entities'):
tweet.entities = json.dumps(tweet.entities)
if hasattr(tweet, 'geo'):
tweet.geo = json.dumps(tweet.geo)
if hasattr(tweet, 'coordinates'):
tweet.coordinates = json.dumps(tweet.coordinates)
if hasattr(tweet, 'extended_entities'):
tweet.extended_entities = json.dumps(tweet.extended_entities)
if hasattr(tweet, 'retweeted_status'):
saveTwitter(tweet.retweeted_status)
tweet.retweeted_status = tweet.retweeted_status.id_str
if hasattr(tweet, 'quoted_status'):
# print tweet.quoted_status
del tweet.quoted_status
# saveTwitter(tweet.quoted_status)
# tweet.quoted_status = tweet.quoted_status.id_str
# place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
if hasattr(tweet, 'place'):
del tweet.place
# for k, v in vars(tweet).items():
# print '%s=%s' % (k, v)
return db_bz.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)
def saveMessage(message):
'''
create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
'''
message.id_str = message.pop('id')
content = storage()
content.type = message.type
content.repo = message.pop('repo')
content.payload = message.pop('payload')
message.content = json.dumps(content)
if message.get('org'):
message.org = json.dumps(message.org)
# return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
m = public_bz.storage()
m.id_str = message.id_str
m.m_type = 'github'
m.m_user_id = message.actor
m.created_at = message.created_at
m.content = message.content
m.text = None
m.href = None
return db_bz.insertIfNotExist(
pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
def saveMessage(message):
'''
create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
'''
message.id_str = message.pop('id')
content = storage()
content.type = message.type
content.repo = message.pop('repo')
content.payload = message.pop('payload')
message.content = json.dumps(content)
if message.get('org'):
message.org = json.dumps(message.org)
# return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
m = public_bz.storage()
m.id_str = message.id_str
m.m_type = 'github'
m.m_user_id = message.actor
m.created_at = message.created_at
m.content = message.content
m.text = None
m.href = None
return db_bz.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
def follow(user_id, god_id, make_sure=True):
'''
create by bigzhu at 15/07/15 14:22:51
modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure
'''
id = db_bz.insertIfNotExist(pg, 'follow_who', {'user_id': user_id, 'god_id': god_id}, "user_id=%s and god_id=%s" % (user_id, god_id))
if id is None and make_sure:
raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
def post(self):
self.set_header("Content-Type", "application/json")
user_id = self.get_secure_cookie("user_id")
for card_number in self.request.body.splitlines():
print "card_number:", card_number
values = {"user_id": user_id, "card_number": card_number}
id = db_bz.insertIfNotExist(pg, "available_card_numbers", values, where=" card_number='%s'" % card_number)
print "insert ", values, "id:", id
self.write(json.dumps({"error": "0"}, cls=public_bz.ExtEncoder))
def saveLast(last_time, last_message_id, user_id):
'''
create by bigzhu at 15/08/16 16:22:39 保存最后一条的message
'''
last_time = int(last_time)
datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True)
id = db_bz.insertIfNotExist(pg, 'last', {'user_id': user_id, 'last_time': datetime_last_time, 'last_message_id': last_message_id}, "user_id=%s" % user_id)
if id is None:
count = pg.update('last', where='last_time< to_timestamp(%s/1000) and user_id=%s' % (last_time, user_id), last_message_id=last_message_id, last_time=datetime_last_time)
return count
return id
def follow(user_id, god_id, make_sure=True):
'''
create by bigzhu at 15/07/15 14:22:51
modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure
'''
id = db_bz.insertIfNotExist(pg, 'follow_who', {
'user_id': user_id,
'god_id': god_id
}, "user_id=%s and god_id=%s" % (user_id, god_id))
if id is None and make_sure:
raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
def post(self):
self.set_header("Content-Type", "application/json")
data = json.loads(self.request.body)
id = db_bz.insertIfNotExist(pg, 'user_info', data, "user_name='%s'" % data['user_name'])
if id is None:
where = "user_name='%s'" % data['user_name']
self.pg.update("user_info", where=where, **data)
id = self.pg.select('user_info', where=where)[0].id
oper.follow(self.current_user, id, make_sure=False)
self.write(json.dumps({'error': '0'}))
def post(self):
self.set_header("Content-Type", "application/json")
data = json.loads(self.request.body)
id = db_bz.insertIfNotExist(pg, 'user_info', data,
"user_name='%s'" % data['user_name'])
if id is None:
where = "user_name='%s'" % data['user_name']
self.pg.update("user_info", where=where, **data)
id = self.pg.select('user_info', where=where)[0].id
oper.follow(self.current_user, id, make_sure=False)
self.write(json.dumps({'error': '0'}))
def saveMessage(message):
'''
create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
'''
message.id_str = message.pop('id')
content = storage()
content.type = message.type
content.repo = message.pop('repo')
content.payload = message.pop('payload')
message.content = json.dumps(content)
if message.get('org'):
message.org = json.dumps(message.org)
return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
def saveLast(last_message_id, user_id):
'''
create by bigzhu at 15/08/16 16:22:39 保存最后一条的message
'''
#last_time = int(last_time)
#datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True)
id = db_bz.insertIfNotExist(pg, 'last', {
'user_id': user_id,
'last_message_id': last_message_id
}, "user_id=%s" % user_id)
if id is None:
count = pg.update('last',
where='last_message_id< %s and user_id=%s' %
(last_message_id, user_id),
last_message_id=last_message_id)
return count
return 1
def saveUser(id, url):
'''
create by bigzhu at 15/07/15 21:27:19 保存github信息
create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
'''
if list(pg.select('github_user', where='id=%s' % id)):
return id
else:
r = requests.get(url)
user = storage(r.json())
del user.url
del user.followers_url
del user.following_url
del user.gists_url
del user.starred_url
del user.subscriptions_url
del user.organizations_url
del user.repos_url
del user.events_url
del user.received_events_url
del user.type
return db_bz.insertIfNotExist(pg, 'github_user', user)
def saveUser(id, url):
'''
create by bigzhu at 15/07/15 21:27:19 保存github信息
create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
'''
if list(pg.select('github_user', where='id=%s' % id)):
return id
else:
r = requests.get(url)
user = storage(r.json())
print user
if hasattr(user, 'url'):
del user.url
del user.followers_url
del user.following_url
del user.gists_url
del user.starred_url
del user.subscriptions_url
del user.organizations_url
del user.repos_url
del user.events_url
del user.received_events_url
del user.type
return db_bz.insertIfNotExist(pg, 'github_user', user)
def insertIfNotExist(*args, **kwargs):
return db_bz.insertIfNotExist(*args, **kwargs)
def getMedia(user_name=None, with_next_url=None, user=None):
if user_name:
user = getUser(user_name)
if user is None:
return
# min_id 会查出大于等于这个id的
try:
medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
if medias:
last_id = medias[0].id
pg.update('instagram_user', where="lower(username)=lower('%s')" % user_name, last_id=last_id)
else:
medias, next_ = api.user_recent_media(with_next_url=with_next_url)
for media in medias:
db_media = storage()
if media.caption:
caption = media.caption.__dict__
caption['user_id'] = caption['user'].id
del caption['user']
else:
caption = ''
db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)
db_media.comment_count = media.comment_count
if media.comments:
media.comments = [d.__dict__ for d in media.comments]
for comment in media.comments:
comment['user'] = comment['user'].__dict__
db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder)
db_media.created_time = media.created_time
# 8小时的问题
db_media.created_time += timedelta(hours=8)
db_media.filter = media.filter
db_media.low_resolution = json.dumps(media.images['low_resolution'].__dict__)
db_media.standard_resolution = json.dumps(media.images['standard_resolution'].__dict__)
db_media.thumbnail = json.dumps(media.images['thumbnail'].__dict__)
db_media.id_str = media.id
db_media.like_count = media.like_count
# likes里有User对象,暂时不存了
#db_media.likes = json.dumps(media.likes)
db_media.link = media.link
db_media.type = media.type
db_media.user_id = user.id
id = db_bz.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
print 'new=', media.id, user.username
if id is not None and len(medias) <= 2: # 新增加消息,微信通知只通知2条以内
openids = public_db.getOpenidsByName('instagram', user.username)
for data in openids:
if caption != '':
text = caption.get('text')
else:
text = ''
wechat_oper.sendInstagram(data.openid, text, media.images['standard_resolution'].url, user.username, id)
# 递归查出
if next_ != with_next_url:
getMedia(with_next_url=next_, user=user)
def insertIfNotExist(*args, **kwargs):
return db_bz.insertIfNotExist(*args, **kwargs)