def process_item(self, item, spider):
logging.debug(item)
insert_sql = "insert into byr_post_reply_count(post_id,reply_count,create_time) values ('%s', '%s', '%s')" % \
(item['id'], item['reply_count'], dateutils.get_now())
mysql_conn_pool = mysqlconnpool.MysqlConnPool()
mysql_conn_pool.execute(insert_sql)
mysql_conn_pool.dispose()
def process_item(self, item, spider):
logging.debug(item)
mysql_conn_pool = mysqlconnpool.MysqlConnPool()
update_sql = "update byr_user_info set face_url = '%s', user_name = '%s', level = '%s', life = '%s', score = '%s', post_count = '%s', modify_time = '%s' where user_id = '%s'" \
% (item['face_url'], item['user_name'], item['level'], item['life'], item['score'], item['post_count'], dateutils.get_now(), item['user_id'])
mysql_conn_pool.execute(update_sql)
mysql_conn_pool.dispose()
return item
def process_item(self, item, spider):
logging.debug(item)
mysql_conn_pool = mysqlconnpool.MysqlConnPool()
is_end = 1
try:
self.save_post(item, mysql_conn_pool)
self.save_author(item, mysql_conn_pool)
is_end = 1
except Exception:
traceback.print_exc()
is_end = 0
finally:
mysql_conn_pool.dispose(is_end)
return item # 返回item，告诉引擎，我已经处理好了，你可以进行下一个item数据的提取了
def get_post(self):
mysql_conn_pool = mysqlconnpool.MysqlConnPool()
select_sql = 'select post_id,link from byr_post_detail order by modify_time desc limit 10'
result = mysql_conn_pool.get_all(select_sql)
mysql_conn_pool.dispose()
return result
def get_user_id(self):
mysql_conn_pool = mysqlconnpool.MysqlConnPool()
select_sql = 'select user_id from byr_user_info'
result = mysql_conn_pool.get_all(select_sql)
mysql_conn_pool.dispose()
return result