def _get_data_from_sina(self):
'''获取新浪微博的数据'''
_url_list = []
_data_list = []
_keywords = get_keywords().split(',')
try:
for keyword in _keywords:
if 'SERVER_SOFTWARE' in os.environ:
url = 'http://api.t.sina.com.cn/trends/statuses.json?count=40&source=%s&trend_name=%s' % (
get_appkey(), keyword)
url = url.encode('utf-8')
url = urllib2.unquote(url)
items = json.loads(urllib2.urlopen(url).read())
else:
items = json.loads(
requests.get(
'http://api.t.sina.com.cn/trends/statuses.json?count=40&source=%s&trend_name=%s'
% (get_appkey(), keyword)).content)
for item in items:
i = Dict(item)
if not i.thumbnail_pic:
continue
_url_id = url2id(i.thumbnail_pic)
_url_list.append(str(_url_id))
_dic = {str(_url_id): item}
_data_list.append(_dic)
except Exception, what:
return what
def add(cls):
'''抓取新浪微博数据'''
items = cls()._get_data_from_sina()
_black_list = BlackListService.get_all_black()
_status = 2 #默认数据是未审核
if not items:
return "None"
for item in items:
i = Dict(item)
u = Dict(i.user)
_is_in_black = False
for _black in _black_list:
if (_black["type"] == 0 and int(_black["value"]) == u.id) or (
_black["type"] == 1 and _black["value"] in i.text):
_is_in_black = True
continue
if _is_in_black:
continue
try:
_img = get_remote_image_width_height(i.thumbnail_pic)
_width = _img[0]
_height = int(_img[1] * 255 / _img[0])
PicsHandler().insert(id=i.id,
user_id=u.id,
text=i.text,
user_name=u.name,
create_date=formattime(i.created_at),
thumbnail_pic=i.thumbnail_pic,
bmiddle_pic=i.bmiddle_pic,
original_pic=i.original_pic,
profile_image_url=u.profile_image_url,
url_id=url2id(i.thumbnail_pic),
source=1,
status=_status,
height=_height,
width=_width)
except:
continue
return "Success %s" % len(items)
def _get_data_from_sina(self):
'''获取新浪微博的数据'''
_url_list = []
_data_list = []
_keywords = get_keywords().split(',')
try:
for keyword in _keywords:
if 'SERVER_SOFTWARE' in os.environ:
url = 'http://api.t.sina.com.cn/trends/statuses.json?count=40&source=%s&trend_name=%s' % (get_appkey(), keyword)
url = url.encode('utf-8')
url = urllib2.unquote(url)
items = json.loads(urllib2.urlopen(url).read())
else:
items = json.loads(requests.get('http://api.t.sina.com.cn/trends/statuses.json?count=40&source=%s&trend_name=%s' % (get_appkey(), keyword)).content)
for item in items:
i = Dict(item)
if not i.thumbnail_pic:
continue
_url_id = url2id(i.thumbnail_pic)
_url_list.append(str(_url_id))
_dic = {str(_url_id):item}
_data_list.append(_dic)
except Exception, what:
return what
def add(cls):
'''抓取新浪微博数据'''
items = cls()._get_data_from_sina()
_black_list = BlackListService.get_all_black()
_status = 2 #默认数据是未审核
if not items:
return "None"
for item in items:
i = Dict(item)
u = Dict(i.user)
_is_in_black = False
for _black in _black_list:
if (_black["type"] == 0 and int(_black["value"]) == u.id) or (_black["type"] == 1 and _black["value"] in i.text):
_is_in_black = True
continue
if _is_in_black:
continue
try:
_img = get_remote_image_width_height(i.thumbnail_pic)
_width = _img[0]
_height = int(_img[1]*255/_img[0])
PicsHandler().insert(id=i.id, user_id=u.id, text=i.text, user_name=u.name, create_date=formattime(i.created_at), thumbnail_pic=i.thumbnail_pic, bmiddle_pic=i.bmiddle_pic, original_pic=i.original_pic, profile_image_url=u.profile_image_url, url_id=url2id(i.thumbnail_pic), source=1, status=_status, height=_height, width=_width)
except:
continue
return "Success %s"%len(items)
