def read_shop_class_from_db():
shop_class = []
db = Database()
shop_class = db.select_from('SELECT * FROM crawler.mt_directory')
db.close()
return shop_class
# 'webloc_geo':'30.3007%2C120.167638%2Cwgs84',
# 'ci':'50',
# ' _lxsdk_s': '1634d0b9358-04-0ab-c9c%7C%7C30'
# }
# user_agent_list = [
# 'Mozilla/5.0(Macintosh;U;IntelMacOSX10_6_8;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50',
# 'Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50',
# 'Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0',
# 'Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)',
# 'Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1',
# 'Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1',
# 'Opera/9.80(Macintosh;IntelMacOSX10.6.8;U;en)Presto/2.8.131Version/11.11'
# ]
db = Database()
down = Downloader(headers=headers_shop, cache=None)
sql = "SELECT SHOP_ID, SECOND_LEVEL_DIRECTORY FROM crawler.mt_meishi where LABEL_IS_CCRAWLED = 0 and FIRST_LEVEL_DIRECTORY = '休闲娱乐' limit 50;"
def update_shop_by_id(data):
sql_hand = 'update crawler.mt_meishi set '
sql_end = ' where SHOP_ID = ' + str(data['SHOP_ID']) + ';'
sql_body = ''
for key, value in data.items():
if data[key] == None:
continue
elif type(data[key]) == int:
sql_body = sql_body + key + ' = ' + str(data[key]) + ', '
else:
sql_body = sql_body + key + ' = ' + "'" + str(
def write_data_to_db():
data = shop_info_list.pop()
db = Database()
db.insert_into(data)