def db_query_extend(self):
one = 0
try:
conn = sqlite3.connect(self.db_path_name, timeout = 2000, check_same_thread = False)
while True:
sql_query_data = ''' select http_url from jd_info where is_exted = 0'''
results = conn.execute(sql_query_data)
one = results.fetchone()[0]
#optimize for database fast process
if one:
if re.match(r'^http://(help|red|tuan|auction|jr|smart|gongyi|app|en|media|m|myjd|chat|read|chongzhi|z|giftcard|fw|you|mobile|me|wiki).jd.com', one) or re.match(r'^http://www.jd.com/compare/', one) or re.match(r'^http://club.jd.com/consultation/', one) :
#print("正在处理：%s [删除]" % ( one) )
sql_delete_date = ''' delete from jd_info where http_url = '%s' ''' % one
conn.execute(sql_delete_date)
conn.commit()
continue
# Got one
sql_update_data = '''update jd_info set acc_time = '%s', is_exted = 1 where http_url = '%s' ''' %(jd_utils.current_time(), one)
conn.execute(sql_update_data)
conn.commit()
break
except Exception as e:
print ('-4--'+str(e)+'--')
finally:
conn.commit()
conn.close()
return one
def db_query_comment(self):
sql_query_data = ''' select http_url from jd_info where is_product = 1 and is_comm = 0'''
one = 0
try:
conn = sqlite3.connect(self.db_path_name, timeout = 2000, check_same_thread = False)
results = conn.execute(sql_query_data)
one = results.fetchone()[0]
if one:
sql_update_data = '''update jd_info set acc_time = '%s', is_comm = 1 where http_url = '%s' ''' %(jd_utils.current_time(), one)
conn.execute(sql_update_data)
conn.commit()
except Exception as e:
print ('-3--'+str(e)+'--')
finally:
conn.close()
return one
def do_db_insert(self, http_url, product = 0, extend = 0, process = 0, comment = 0):
sql_query_data = ''' select count(*) from jd_info where http_url = '%s' ''' % http_url
try:
conn = sqlite3.connect(self.db_path_name, timeout = 2000, check_same_thread = False)
total = conn.execute(sql_query_data)
if total.fetchone()[0] > 0:
return
sql_insert_data = '''insert into jd_info(http_url,acc_time,is_product,is_exted,is_cons,is_comm) values ('%s', '%s', '%d', '%d', '%d','%d')'''%(http_url,jd_utils.current_time(),product,extend,process,comment)
conn.execute(sql_insert_data)
conn.commit()
except Exception as e:
print ('-6--'+str(e)+'--')
finally:
conn.close()
def db_update_extend(self, http_url, extend = 1 ):
sql_insert_data = '''update jd_info set acc_time = '%s', is_exted = '%d' where http_url = '%s' ''' %(jd_utils.current_time(), extend , http_url)
try:
conn = sqlite3.connect(self.db_path_name, timeout = 2000, check_same_thread = False)
conn.execute(sql_insert_data)
conn.commit()
except Exception as e:
print ('-9--'+str(e)+'--')
finally:
conn.close()
def encasp_toutiao(self, tt_item):
web_msg = u"【开发者头条】" + u"标题：" + tt_item['TITLE'].decode('utf-8') + \
u"\n主题：" + tt_item['SUMMARY'].decode('utf-8') + \
u"\n链接：" + tt_item['LINK'].decode('utf-8') + \
u"\n[Nicol's Robot %s]" % (jd_utils.current_time())
return web_msg