def wrapper(self, *args, **kwargs):
self.set_header("Content-Type", "application/json")
try:
method(self, *args, **kwargs)
except Exception:
print public_bz.getExpInfoAll()
self.write(json.dumps({'error': public_bz.getExpInfo()}))
def main(god, wait):
'''
create by bigzhu at 15/07/15 17:54:08 取github
modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
'''
god_name = god.name
github_name = god.github['name']
god_id = god.id
etag = god.github.get('sync_key')
github_user = getGithubUser(github_name, god_name)
if github_user is None:
return
headers = {'If-None-Match': etag}
try:
r = requests.get('https://api.github.com/users/%s/events' %
github_name,
headers=headers,
params=params)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
etag = r.headers['etag']
limit = r.headers['X-RateLimit-Remaining']
if limit == '0':
return
for message in r.json():
message = storage(message)
saveMessage(god_name, github_name, god_id, message)
# oper.noMessageTooLong('github', github_name)
saveUser(god_name, github_name, github_user, etag) # 为了update etag
if r.status_code == 404:
# public_db.sendDelApply('github', god_name, github_name, '404')
god_oper.delNoName('github', god_name)
def query(*args, **kwargs):
try:
return db.query(*args, **kwargs)
except Exception:
print public_bz.getExpInfoAll()
print args[0]
print kwargs
def wrapper(self, *args, **kwargs):
try:
return method(self, *args, **kwargs)
except OfficialAPIError:
print public_bz.getExpInfoAll()
settings, self.wechat = initWechat(self.settings)
return method(self, *args, **kwargs)
def wrapper(self, *args, **kwargs):
try:
return method(self, *args, **kwargs)
except(psycopg2.OperationalError, psycopg2.InterfaceError, psycopg2.DatabaseError):
print public_bz.getExpInfoAll()
self.pg.connect()
time.sleep(5)
return wrapper(self, *args, **kwargs)
print '重新连接数据库'
def getClickCount():
try:
f = open(CLICK_COUNT, 'a+')
global click_count
content = f.read()
if content != '':
click_count = json.loads(content)
f.close()
except IOError:
print public_bz.getExpInfoAll()
def getClickCount():
try:
f = open(CLICK_COUNT, 'a+')
global click_count
content = f.read()
if content != '':
click_count = json.loads(content)
f.close()
except IOError:
print public_bz.getExpInfoAll()
def geetest_register(self):
apireg = "http://api.geetest.com/register.php?"
regurl = apireg + "gt=%s" % self.CAPTCHA_ID
try:
result = urllib2.urlopen(regurl, timeout=2).read()
except:
result = ""
print regurl
print public_bz.getExpInfoAll()
return result
def getUserEvent(user_name, etag):
'''
create by bigzhu at 15/07/15 17:54:08 取github
modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
'''
headers = {'If-None-Match': etag}
try:
r = requests.get('https://api.github.com/users/%s/events' % user_name,
headers=headers)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
messages = r.json()
if not messages:
delGithubUser(user_name)
# 没有这个github用户,取消
return
actor = messages[0]['actor']
# actor不定是作者名字，有可能org才是
if actor['login'].lower() == user_name.lower():
the_user = actor
else:
org = messages[0]['org']
if org['login'].lower() == user_name.lower():
the_user = org
# 如果是org，那么url不同
the_user['url'] = "https://api.github.com/users/" + user_name
else:
raise "in this github can't find user_name=%s"
user_id = saveUser(the_user['id'], the_user['url'])
# 更新etag
etag = r.headers['etag']
updateEtag(user_name, etag)
for i in r.json():
i['actor'] = user_id
message = storage(i)
message.created_at = time_bz.unicodeToDateTIme(message.created_at)
message.created_at += timedelta(hours=8)
saveMessage(copy.deepcopy(message))
#id = saveMessage(copy.deepcopy(message))
# if id is not None:
# text = formatInfo(message)
# print text
# openids = public_db.getOpenidsByName('github', user_name)
# for data in openids:
# wechat_oper.sendGithub(data.openid, text, user_name, id)
if r.status_code == 404:
delGithubUser(user_name)
else:
print r.status_code
def getUserEvent(user_name, etag):
'''
create by bigzhu at 15/07/15 17:54:08 取github
modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
'''
headers = {'If-None-Match': etag}
try:
r = requests.get('https://api.github.com/users/%s/events' % user_name, headers=headers)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
messages = r.json()
if not messages:
delGithubUser(user_name)
# 没有这个github用户,取消
return
actor = messages[0]['actor']
# actor不定是作者名字，有可能org才是
if actor['login'].lower() == user_name.lower():
the_user = actor
else:
org = messages[0]['org']
if org['login'].lower() == user_name.lower():
the_user = org
# 如果是org，那么url不同
the_user['url'] = "https://api.github.com/users/" + user_name
else:
raise "in this github can't find user_name=%s"
user_id = saveUser(the_user['id'], the_user['url'])
# 更新etag
etag = r.headers['etag']
updateEtag(user_name, etag)
for i in r.json():
i['actor'] = user_id
message = storage(i)
message.created_at = time_bz.unicodeToDateTIme(message.created_at)
message.created_at += timedelta(hours=8)
saveMessage(copy.deepcopy(message))
#id = saveMessage(copy.deepcopy(message))
# if id is not None:
# text = formatInfo(message)
# print text
# openids = public_db.getOpenidsByName('github', user_name)
# for data in openids:
# wechat_oper.sendGithub(data.openid, text, user_name, id)
if r.status_code == 404:
delGithubUser(user_name)
else:
print r.status_code
def getHtmlContent(name):
'''
取得对应名字的 html 文件的内容
'''
try:
name_file = open(HTML_PATH + name + '.html', 'r')
content = name_file.read()
name_file.close()
return content
except IOError:
print public_bz.getExpInfoAll()
return '0'
def getHtmlContent(name):
'''
取得对应名字的 html 文件的内容
'''
try:
name_file = open(HTML_PATH + name + '.html', 'r')
content = name_file.read()
name_file.close()
return content
except IOError:
print public_bz.getExpInfoAll()
return '0'
def getKeyNames():
try:
f = open('./key_name', 'a+')
global key_names
global key_names_sorted
content = f.read()
if content != '':
key_names = json.loads(content)
popSome()
key_names_sorted = sorted(key_names.items(), key=lambda by: by[1], reverse=True)
f.close()
except IOError:
print public_bz.getExpInfoAll()
def check():
'''
create by bigzhu at 15/07/31 14:28:30
'''
users = pg.select('user_info', what='instagram', where="instagram is not null and instagram!=''")
print len(users)
for user in users:
if user.instagram and user.instagram != '':
print 'check instagram %s' % user.instagram
try:
getMedia(user.instagram)
except Exception:
print public_bz.getExpInfoAll()
def wrapper(self, *args, **kwargs):
try:
return method(self, *args, **kwargs)
# except(psycopg2.OperationalError, psycopg2.InterfaceError, psycopg2.DatabaseError):
# except(psycopg2.InterfaceError, psycopg2.DatabaseError):
except (psycopg2.InterfaceError, psycopg2.OperationalError):
print public_bz.getExpInfoAll()
print args
print kwargs
connect()
time.sleep(5)
print '重新连接数据库'
return wrapper(self, *args, **kwargs)
def getKeyNames():
try:
f = open('./key_name', 'a+')
global key_names
global key_names_sorted
content = f.read()
if content != '':
key_names = json.loads(content)
popSome()
key_names_sorted = sorted(key_names.items(),
key=lambda by: by[1],
reverse=True)
f.close()
except IOError:
print public_bz.getExpInfoAll()
def wrapper(self, *args, **kwargs):
openid = self.get_secure_cookie("openid")
if openid is None:
# 连openid 都没有,首先要获取 openid
params = {
"appid": self.settings['appid'],
# "redirect_uri": "http://" + self.settings['domain'] + "/setOpenid?url=/" + self.__class__.__name__,
# "redirect_uri": "http://" + self.settings['domain'] + "/setOpenid?url=" + self.request.uri,
"redirect_uri": "http://" + "admin.hoywe.com/" + self.settings['suburl'] + "/?url=" + self.request.uri,
"response_type": "code",
"scope": "snsapi_base",
}
auth_url = "https://open.weixin.qq.com/connect/oauth2/authorize?%s#wechat_redirect"
auth_url = auth_url % urllib.urlencode(params)
self.redirect(auth_url)
return
else:
#exists_users = list(self.pg.db.select('wechat_user', where="openid='%s'" % openid))
# if not exists_users:
try:
wechat_user_info = self.wechat.get_user_info(openid, lang='zh_CN')
except OfficialAPIError as e:
print public_bz.getExpInfoAll()
self.clear_cookie(name='openid')
error = public_bz.getExpInfo()
if error.find('40001') != -1:
raise e
else:
error_info = '''
<html>
<script type="text/javascript">
alert("微信服务器异常，请关闭后，重新打开");
WeixinJSBridge.call('closeWindow');
</script>
</html>
'''
self.write(error_info)
return
# 没有关注的,跳转到配置的关注页面
if wechat_user_info['subscribe'] == 0:
self.redirect('http://' + self.settings["domain"] + self.settings["subscribe"])
return
# else:
# print 'add user'
# self.pg.db.insert('wechat_user', **wechat_user_info)
return method(self, *args, **kwargs)
def getCountryCodeByIP(ip):
'''
根据 ip 地址取到所属的国家编码
modify by bigzhu at 15/05/17 11:28:52 add test
>>> getCountryCodeByIP('www.bigzhu.org')
u'US'
'''
country_code = "not found"
url = 'http://freegeoip.net/json/' + ip
try:
info = urllib2.urlopen(url, timeout=10).read()
info = json.loads(info)
if info.get('country_code') != '':
country_code = info.get('country_code')
except Exception:
print public_bz.getExpInfoAll()
return country_code
def getGithubUser(github_name, god_name):
try:
r = requests.get('https://api.github.com/users/%s' % github_name,
params=params)
github_user = r.json()
message = github_user.get('message')
if message == 'Not Found':
god_oper.delNoName('github', god_name)
return
saveUser(god_name, github_name, github_user)
return github_user
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
except ValueError:
print public_bz.getExpInfoAll()
return
def callGetMeidaApi(user_name, offset=0, limit=20):
api_key = 'fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4'
params = {'api_key': api_key,
'offset': offset,
'limit': limit,
}
url = '''http://api.tumblr.com/v2/blog/%s.tumblr.com/posts''' % user_name
try:
r = requests.get(url, params=params)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
medias = r.json()
return medias
else:
print r.status_code
def getUser(user_name, always_check=False):
'''
modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
'''
users = list(
pg.select('instagram_user',
where="lower(username)=lower('%s')" % user_name))
if users and not always_check:
return users[0]
else:
user = None
try:
for this_user in api.user_search(user_name, 10):
if this_user.username.lower() == user_name.lower():
user = this_user
break
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
return
# 如果没有这个用户
if user is None:
public_db.delNoName('instagram', user_name)
return
try:
user = api.user(user.id)
except (instagram.bind.InstagramAPIError,
instagram.bind.InstagramClientError):
# 通常是没有访问权限
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
db_user = storage()
db_user.id_str = user.id
db_user.username = user.username
db_user.full_name = user.full_name
db_user.profile_picture = user.profile_picture
db_user.bio = user.bio
db_user.website = user.website
db_user.counts = json.dumps(user.counts)
#pg.insert('instagram_user', **db_user)
pg.insertOrUpdate(pg, 'instagram_user', db_user,
"id_str='%s'" % db_user.id_str)
#db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
return getUser(user_name)
def getFollowedCount(tumblr_name):
'''
没成功, 取不到
'''
params = {
'api_key': API_KEY,
'blog-identifier': '%s.tumblr.com' % tumblr_name
}
url = '''http://api.tumblr.com/v2/blog/%s.tumblr.com/followers''' % tumblr_name
r = requests.get(url, params=params)
print r
if r.status_code == 200:
try:
medias = r.json()
return medias
except Exception:
print 'r=', r
print public_bz.getExpInfoAll()
print r.status_code
def callGetMeidaApi(god_name, offset=0, limit=20):
params = {
'api_key': API_KEY,
'offset': offset,
'limit': limit,
}
url = '''http://api.tumblr.com/v2/blog/%s.tumblr.com/posts''' % god_name
r = requests.get(url, params=params)
if r.status_code == 200:
try:
medias = r.json()
return medias
except Exception:
print 'r=', r
print public_bz.getExpInfoAll()
return
elif r.status_code == 429:
raise Exception('达到最大访问次数')
else:
print r.status_code
def getUser(user_name, always_check=False):
'''
modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
'''
users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name))
if users and not always_check:
return users[0]
else:
user = None
try:
for this_user in api.user_search(user_name, 10):
if this_user.username.lower() == user_name.lower():
user = this_user
break
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
return
# 如果没有这个用户
if user is None:
public_db.delNoName('instagram', user_name)
return
try:
user = api.user(user.id)
except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError):
# 通常是没有访问权限
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
db_user = storage()
db_user.id_str = user.id
db_user.username = user.username
db_user.full_name = user.full_name
db_user.profile_picture = user.profile_picture
db_user.bio = user.bio
db_user.website = user.website
db_user.counts = json.dumps(user.counts)
#pg.insert('instagram_user', **db_user)
pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str)
#db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
return getUser(user_name)
def main(user_name=None):
user = getUser(user_name, always_check=True)
if user is None:
# 用户都没有，不用往下了
return
try:
# https://api.instagram.com/v1/users/1337827037/media/recent/?access_token=1337827037.933ab14.2a607a5fc0534f9f9900e75196a2dfbb&min_id=1054034416535329463_1337827037
# 即使设置了min_id,instagram还是会把当前这条min_id返回来，简直了
#medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
medias = callGetMeidaApi(user.id, user.last_id)
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
if medias is None:
return
saveLastId(user, medias)
if len(medias['data']) != 0:
print len(medias['data'])
saveMedias(user, medias)
def callGetMeidaApi(user_id=None, min_id=None, next_url=None):
params = {'count': 9999,
'access_token': access_token,
}
if min_id:
params['min_id'] = min_id
if user_id:
url = '''https://api.instagram.com/v1/users/%s/media/recent''' % user_id
try:
if next_url:
r = requests.get(next_url)
else:
r = requests.get(url, params=params)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
medias = r.json()
return medias
else:
print r.status_code
def main(user_name=None):
user = getUser(user_name, always_check=True)
if user is None:
# 用户都没有，不用往下了
return
try:
# https://api.instagram.com/v1/users/1337827037/media/recent/?access_token=1337827037.933ab14.2a607a5fc0534f9f9900e75196a2dfbb&min_id=1054034416535329463_1337827037
# 即使设置了min_id,instagram还是会把当前这条min_id返回来，简直了
#medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
medias = callGetMeidaApi(user.id, user.last_id)
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
if medias is None:
return
saveLastId(user, medias)
if len(medias['data']) != 0:
print len(medias['data'])
saveMedias(user, medias)
def getContent(name):
if name in NOT_IN:
return '# 不要乱访问哦! 这不是你有权限可以看的东西!'
try:
name_file = open(MD_PATH + name + '.md', 'r')
content = name_file.read()
if 'status: draft' in content:
return '# 这是一个机密文件, 不允许查看!'
name_file.close()
return content
except IOError:
print(public_bz.getExpInfoAll())
return '0'
def callGetMeidaApi(user_id=None, min_id=None, next_url=None):
params = {
'count': 9999,
'access_token': access_token,
}
if min_id:
params['min_id'] = min_id
if user_id:
url = '''https://api.instagram.com/v1/users/%s/media/recent''' % user_id
try:
if next_url:
r = requests.get(next_url)
else:
r = requests.get(url, params=params)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
return
if r.status_code == 200:
medias = r.json()
return medias
else:
print r.status_code
# oper.noMessageTooLong(M_TYPE, tumblr_name)
saveUser(god_name, tumblr_name, tumblr_user)
def loop(god_name=None, wait=None):
'''
'''
gods = social_sync.getSocialGods('tumblr', god_name)
for god in gods:
main(god, wait)
if __name__ == '__main__':
if len(sys.argv) == 2:
god_name = (sys.argv[1])
loop(god_name)
exit(0)
while True:
try:
loop(wait=True)
except requests.exceptions.ConnectionError:
print public_bz.getExpInfoAll()
except requests.exceptions.ChunkedEncodingError as e:
print e
except requests.exceptions.ReadTimeout as e:
print e
except NoUser as e:
print e
print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
time.sleep(1200)
def getMedia(user_name=None, with_next_url=None, user=None):
if user_name:
user = getUser(user_name)
if user is None:
return
# min_id 会查出大于等于这个id的
try:
medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
except instagram.bind.InstagramClientError:
print public_bz.getExpInfoAll()
public_db.delNoName('instagram', user_name)
return
if medias:
last_id = medias[0].id
pg.update('instagram_user', where="lower(username)=lower('%s')" % user_name, last_id=last_id)
else:
medias, next_ = api.user_recent_media(with_next_url=with_next_url)
for media in medias:
db_media = storage()
if media.caption:
caption = media.caption.__dict__
caption['user_id'] = caption['user'].id
del caption['user']
else:
caption = ''
db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)
db_media.comment_count = media.comment_count
if media.comments:
media.comments = [d.__dict__ for d in media.comments]
for comment in media.comments:
comment['user'] = comment['user'].__dict__
db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder)
db_media.created_time = media.created_time
# 8小时的问题
db_media.created_time += timedelta(hours=8)
db_media.filter = media.filter
db_media.low_resolution = json.dumps(media.images['low_resolution'].__dict__)
db_media.standard_resolution = json.dumps(media.images['standard_resolution'].__dict__)
db_media.thumbnail = json.dumps(media.images['thumbnail'].__dict__)
db_media.id_str = media.id
db_media.like_count = media.like_count
# likes里有User对象,暂时不存了
#db_media.likes = json.dumps(media.likes)
db_media.link = media.link
db_media.type = media.type
db_media.user_id = user.id
id = db_bz.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
print 'new=', media.id, user.username
if id is not None and len(medias) <= 2: # 新增加消息,微信通知只通知2条以内
openids = public_db.getOpenidsByName('instagram', user.username)
for data in openids:
if caption != '':
text = caption.get('text')
else:
text = ''
wechat_oper.sendInstagram(data.openid, text, media.images['standard_resolution'].url, user.username, id)
# 递归查出
if next_ != with_next_url:
getMedia(with_next_url=next_, user=user)
