def recordBanner_pic(db_app, banners):
cursor = db_app.get_cursor()
try:
for banner in banners:
url = banner['banner']
text = banner['text']
cursor.execute(
"SELECT id FROM banner where pic_url='%s' and title='%s'" %
(url, text))
result = cursor.fetchone()
if result is None:
t = time.time()
ts = int(round(t * 1000))
sql = "INSERT INTO banner (pic_url, title, `timestamp`, `position`, `resource_type`) values ('%s', '%s', %d, 'index', 1)" % (
url, text, ts)
# print sql
cursor.execute(sql)
else:
print "banner exists..."
db_app.commit()
except Exception, e:
log.error(e.message)
log.error(traceback.format_exc())
def recordBanner(db_app, banners):
cursor = db_app.get_cursor()
try:
modules = get_modules(db_app)
str = ','.join(banners)
# print str
cursor.execute(
"DELETE FROM resource_banner where resource_id not in (%s)" % str)
for bn in banners:
banner = int(bn)
for module in modules:
status = 0
if module == 9:
status = 1
cursor.execute(
"SELECT id FROM resource_banner where `position`=%d and resource_id=%d"
% (module, banner))
result = cursor.fetchone()
if result is None:
t = time.time()
ts = int(round(t * 1000))
sql = "INSERT INTO resource_banner (status, `timestamp`, `position`, resource_id) values (%d, %d,%d,%d)" % (
status, ts, module, banner)
print sql
cursor.execute(sql)
else:
print "banner exists ..."
db_app.commit()
except Exception, e:
log.error(e.message)
log.error(traceback.format_exc())
sql = "INSERT INTO brand_opinion (`day`,full_num,num,brand_id,`timestamp`)" \
" VALUES ('%s',%d,%d,%d,%d)" % (str_time, value1, value2, brand_id, ts)
# print sql
cursor.execute(sql)
else:
id = result[0]
# print id
#cursor.execute("UPDATE brand_opinion set full_num=%d, num=%d, `timestamp`=%d where id=%d and `day`='%s'" % (
# value1, value2, ts, id, str_time))
if __name__ == '__main__':
# s = ''
# html, img_list, author, video_list = sync.get_href('http://www.qicycling.cn/2927.html')
# print html
# print time.time()
# cursor_wp = db_wp.get_cursor()
#print datetime.datetime.now().strftime('%Y-%m-%d')
cursor_app = db_app.get_cursor()
# import_brand(cursor_app)
# db_app.commit()
b_l = get_brand_list(cursor_app)
print b_l
get_google_trend(cursor_app, b_l)
#
# get_google_trend_brand(cursor_app, '3T')
db_app.commit()
def recordArticle(db_app, articles):
cursor_app = db_app.get_cursor()
cursor_wp = db_wp.get_cursor()
try:
for article in articles:
article_id = int(article['article_id'])
img_url = article['src']
title = article['alt']
description = article['text']
href = article['href']
print("--------- todo article:%d" % article_id)
# check unsync module
if is_unsync_category(cursor_wp, article_id):
print("------- article:{} is unsync module, continue".format(
article_id))
continue
cursor_app.execute("SELECT status FROM resource where id=%d" %
article_id)
result = cursor_app.fetchone()
if result is None:
html, img_list, author, v_list, t_ = get_href(href)
# 时间从 db_wp 读取 wp_posts.id =
if len(img_list) <= 0:
img_ = Image.open(requests.get(img_url, stream=True).raw)
width, height = img_.size
pinfo = {
'url': img_url,
'width': int(width),
'height': int(height)
}
img_list.append(pinfo)
cursor_wp.execute(
"SELECT UNIX_TIMESTAMP(post_date) FROM wp_posts where id=%d"
% article_id)
result = cursor_wp.fetchone()
ts = result[0]
# print article_id, img_list, author, title, description, ts
h5 = save_h5(html, article_id)
sql = "INSERT INTO resource (id, title, description, status, h5url, is_vip, publisher, type,TIMESTAMP)" \
" VALUES (%d, '%s', '%s', 1, '%s', 0, '%s', 0, %d)" % (
article_id, title, description, h5, author, ts * 1000)
cursor_app.execute(sql)
if (len(v_list) > 0):
v_url = quote(v_list[0])
# v_url = quote(v_list[0])
sql = "UPDATE resource set video_url='%s' where id=%d" % (
v_url, article_id)
# print sql
cursor_app.execute(sql)
for i in xrange(len(img_list)):
img = img_list[i]
# surl = quote(img['url'])
surl = img['url']
sql = "INSERT INTO resource_image (resource_id, height, `length`, url) values (%d,%d,%d,'%s')" % (
article_id, int(img['width']), int(
img['height']), surl)
cursor_app.execute(sql)
# post to
post_to_search(article_id, title, description, h5, author)
else:
# print("article_id:%d already exists."%article_id)
continue
# 修改赛事类型
add_resource_module(cursor_wp, cursor_app, article_id)
print("--------- done article:%d" % article_id)
db_app.commit()
db_wp.commit()
except Exception, e:
log.error(e.message)
log.error(traceback.format_exc())