def save_question(cols):
mysql_conn = get_mysql_connection()
#
sql, vals = insert_sql('question_db_offline.wln100_question_20170919',
cols,
ignore=True)
execute(mysql_conn, sql, values=vals)
def record_questions(rows):
mysql_conn = get_mysql_connection()
for row in rows:
html_string = row[1]
spider_url = row[2]
info = json.loads(row[3])
# if is_archived(spider_url):
# continue
logging.info(spider_url)
try:
cols = parser.parse(html_string, spider_url, info)
except Exception as err:
logging.error('[parser.parse] {}, {}'.format(err, spider_url))
continue
if not cols:
continue
# print(json.dumps(cols, indent=4, ensure_ascii=False))
sql, vals = insert_sql(
'question_db_offline.manfen5_zujuan_question_20161205',
cols,
ignore=True)
execute(mysql_conn, sql, values=vals)
mysql_conn.commit()
def record_questions(rows):
mysql_conn = get_mysql_connection()
for row in rows:
js = json.loads(row[1])
spider_url = row[2]
aft_subj_id = row[3]
try:
cols = parser.parse(spider_url, js, aft_subj_id)
except Exception as err:
logging.error('[parser.parse] {}, {}'.format(err, spider_url))
continue
# print(json.dumps(cols, indent=4, ensure_ascii=False))
sql, vals = insert_sql('question_db_offline.17zuoye_question_20160719',
cols,
ignore=True)
execute(mysql_conn, sql, values=vals)
def record_questions(rows):
mysql_conn = get_mysql_connection()
for row in rows:
html_string = row[1]
spider_url = row[2]
try:
cols = parser.parse(html_string, spider_url)
except Exception as err:
logging.error('[parser.parse] {}, {}'.format(err, spider_url))
continue
if not cols:
continue
# print(json.dumps(cols, indent=4, ensure_ascii=False))
sql, vals = insert_sql('question_db_offline.gzywtk_question_20161109',
cols,
ignore=True)
execute(mysql_conn, sql, values=vals)
def record_questions(rows):
mysql_conn = get_mysql_connection()
for row in rows:
try:
js = json.loads(row[1])
spider_url = row[2]
cols = parser.parse(js, spider_url)
except Exception as err:
logging.error('[parser.parse] {}, {}'.format(err, spider_url))
continue
if not cols:
continue
# print(json.dumps(cols, indent=4, ensure_ascii=False))
logging.info(spider_url)
sql, vals = insert_sql('question_db_offline.vko_question_20161116',
cols,
ignore=True)
execute(mysql_conn, sql, values=vals)