from legacy.crawler import crawler
import os
url = "https://cfobase.com/"
output_dir = os.path.join('.', 'data', 'cfobase.com')
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': '*',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Host': 'cfobase.com',
'Connection': 'keep-alive',
'Cookie':
'_mkto_trk=id:497-ITQ-712&token:_mch-bromium.com-1545360431008-12984; _biz_sid=919162; _ga=GA1.2.1945164141.1545360432; _gid=GA1.2.949409536.1545360432; _biz_uid=a18ee78fd1664b1ecb82da38af7ca704; _biz_flagsA=%7B%22Version%22%3A1%2C%22Mkto%22%3A%221%22%2C%22XDomain%22%3A%221%22%7D; _biz_nA=4; _biz_pendingA=%5B%5D',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
if __name__ == "__main__":
from legacy import common_crawl
wpc = crawler.WordPressCrawler(url, headers, output_dir)
common_crawl(wpc)
from legacy.crawler import crawler
import os
url = "https://www.vipre.com"
output_dir = os.path.join('.', 'data', 'www.vipre.com')
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': '*',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Cookie':
'_ga=GA1.2.1644535100.1541147443; __adroll_fpc=3a119bf44db7ee75d21bdc4bd8006d25; __idcontext=eyJjb29raWVJRCI6IjZFM0I3VUNSNDVOWVZHT1o3WTJGSDZOT1dXRUtFMkM0TlhKNUdBVDdIVkdRPT09PSIsImRldmljZUlEIjoiNkUzQjdVQ1I0QjNKRlg2WFZZSVdMNjcyU1M0SllNU0dKTEc1T0FDR0c0RUE9PT09IiwiaXYiOiI1SVdKNlpMUlBKN0lNNlcyVkpWMjZQRzJKST09PT09PSIsInYiOjF9; _gid=GA1.2.1952222110.1541383857; _gcl_au=1.1.535369970.1541532633; __ar_v4=R5EX2LAD7FAOVD6PWNPH6O%3A20181102%3A10%7CN7AGIAEPRZDM5FMAGV2QUY%3A20181102%3A10%7C53FLNYE57ZE4ZDAHOYANNY%3A20181102%3A10',
'Host': 'www.vipre.com',
'Connection': 'keep-alive',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/70.0.3538.67 Chrome/70.0.3538.67 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
if __name__ == "__main__":
from legacy import common_crawl
wpc = crawler.WordPressCrawler(url,
headers,
output_dir,
crawl_rate=100,
verify_ssl=False)
common_crawl(wpc)
from legacy.crawler import crawler
import os
url = "https://unwire.pro"
output_dir = os.path.join('.', 'data', 'unwire.pro')
headers = {
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding':
'*',
'Accept-Language':
'zh-CN,zh;q=0.8',
'Connection':
'keep-alive',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/70.0.3538.67 Chrome/70.0.3538.67 Safari/537.36',
}
if __name__ == "__main__":
from legacy import common_crawl
wpc = crawler.WordPressCrawler(url, headers, output_dir, crawl_rate=5)
common_crawl(wpc)
def crawl(url, output_dir, crawl_rate=25):
wpc = crawler.WordPressCrawler(url, common_header, output_dir, crawl_rate)
common_crawl(wpc)
