def run(self): start_mark = False crawler = Crawler() while self.flag: try: url = 'http://hw.venndata.cn/proxy?num={num}'.format( num=self.num) response, _ = crawler.crawl(url=url) html = response.text if html: data = json.loads(html)['data'] proxy_list = data.split('|') if len(proxy_list) > 500: old_len = len(PROXY_LIST) PROXY_LIST.extend(proxy_list) PROXY_LIST[0:old_len] = [] if not start_mark: log.critical("代理启动成功!获取代理%s" % len(proxy_list)) start_mark = True except Exception as e: log.error('生成 Proxy Failed' + str(e)) time.sleep(self.interval) log.info('代理关闭') return
def crawl(base: str): clr = Crawler(base) pages = clr.crawl(depth=0) pages = { 'urls': pages, } results = json.dumps(pages) return jsonify(results)
parser.add_argument( '-u', '--url', required=True, type=str, help='For example => http://yoyowallet.com/') parser.add_argument( '-o', '--out', required=True, type=str, help='You have to enter a valid file address') parser.add_argument( '-t', '--type', required=True, type=FileType.from_string, choices=list(FileType), help='You have to choose one of theme => csv or xml') args = parser.parse_args() crawler = Crawler(str(args.url)) task = asyncio.Task(crawler.crawl()) loop = asyncio.get_event_loop() print(f'\n{30*"*"} crawler is working {30*"*"}\n\n') loop.run_until_complete(task) loop.close() result = task.result() print(f'\n\n{30*"*"} crawling was done {30*"*"}\n\n') Export().print(str(args.type), result, str(args.out)) print(f'{30*"*"} output save on {args.out} {30*"*"}\n')
from crawl import Crawler from house import House from connect2db import DbConnector from config import HEADERS import preprocess from connect2db import action_type if __name__ == "__main__": """main""" mysql_connector = DbConnector() shcrawler = Crawler(HEADERS, 'sh') rg_urls = shcrawler.composeurl(1, 20) for region, urls in rg_urls.iteritems(): for url in urls: res = shcrawler.parse(shcrawler.crawl(url)) for i in res: i.region = region preprocess.main(i) try: action, update_fields = mysql_connector.search(i) if action == action_type.insert: mysql_connector.insert(i) elif action == action_type.update: mysql_connector.update(i, update_fields) elif action == action_type.none: print "already inserted and up-to-date" except Exception as e: print e.message + i.title_url traceback.print_exc() finally:
user_name='root', password='', dbname='taobao_sf') rows = db.execute( 'SELECT distinct(itemId) FROM taobao_sf.sf_list_itemid').fetchall() for row in rows: item_id = row[0] print(item_id) url = 'https://h5api.m.taobao.com/h5/mtop.taobao.govauctionmtopcommonservice.getfrontcategory/1.0/?jsv=2.4.5&appKey=12574478&t=1570096614606&api=mtop.taobao.govauctionmtopcommonservice.getfrontcategory' headers = { 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36' } crawler = Crawler() res, session = crawler.crawl(url=url, headers=headers) cookies = res.cookies.get_dict() m_h5_tk = cookies['_m_h5_tk'] app_key = '12574478' data = '{"itemId":"%s"}' % item_id sign, t = get_sign(m_h5_tk, app_key, data) params = { 'jsv': '2.4.2', 'appKey': app_key, 't': t, 'sign': sign, 'api': 'mtop.taobao.GovauctionMTopDetailService.queryHttpsItemDetail', 'v': '2.0', 'ecode': '0', 'type': 'jsonp', 'dataType': 'jsonp',
cate_name = cate_info['name'] print(cate_name) page = 1 while True: url = 'https://h5api.m.taobao.com/h5/mtop.taobao.govauctionmtopcommonservice.getfrontcategory/1.0/?jsv=2.4.5&appKey=12574478&t=1570096614606&api=mtop.taobao.govauctionmtopcommonservice.getfrontcategory' headers = { 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36' } crawler = Crawler() proxy_ip = random.choice(PROXY_LIST) proxies = { "https": 'https://' + proxy_ip, "http": 'http://' + proxy_ip } res, session = crawler.crawl(url=url, headers=headers, proxies=proxies) cookies = res.cookies.get_dict() m_h5_tk = cookies['_m_h5_tk'] app_key = '12574478' data = '{"city":"%s","pageNo":%s,"pageSize":99,"orderId":"0","categoryId":"%s"}' % ( city, str(page), cate_id) sign, t = get_sign(m_h5_tk, app_key, data) params = { 'jsv': '2.4.5', 'appKey': app_key, 't': t, 'sign': sign, 'api': 'mtop.taobao.govauction.sfsearchlist', 'v': '1.0', 'H5Request': 'true', 'type': 'jsonp',
from crawl import Crawler import json from common import get_regex from tools.mysql_operator import MySqlOperator url = 'https://sf.taobao.com/item_list.htm?city=&province=%D5%E3%BD%AD' crawler = Crawler() res, session = crawler.crawl(url=url, encoding='gbk') raw_data = get_regex( r'<script id="sf-item-list-data" type="text/json">([\S\s]*?)</script>', res.text, 1) jdata = json.loads(raw_data) data_list = list() for item in jdata['data']: item_info = {'id': item['id'], 'title': item['title']} data_list.append(item_info) db = MySqlOperator(server='127.0.0.1', user_name='root', password='', dbname='taobao_sf') db.bulk_insert('test_tb', data_list)