def main(model, dummy):
if len(model) == 1:
model = model[0]
dispatcher = Dispatcher()
if model not in available_models:
print('Model', model, 'is not in the folder "scrape_models".')
print('These models are available:')
pprint.pprint(available_models, compact=True)
return
imported = vars(importlib.import_module(f'scrape_models.{model}')).values()
scrape_models = [model for model in imported if type(model) == ScrapeModel]
dispatcher.add_scraper(scrape_models, dummy=dummy)
dispatcher.run()
from modelscraper.dispatcher import Dispatcher import nu_nl import metronieuws import parool import volkskrant disp = Dispatcher() disp.add_scraper([volkskrant.volkskrant]) disp.run()
product = Phase(
sources=sources,
templates=[
Template(
name='product',
db_type='mongo_db',
db='makro',
table='products',
attrs=[
Attr(name='name', selector='h1', func='sel_text'),
Attr(name='price_gross',
selector='.price-gross',
func='sel_text'), # kws={'replacers': '€ '}),
Attr(name='price_net', selector='.price-net',
func='sel_text'), # kws={'replacers': '€ '}),
Attr(name='sku', selector='.articlenumber', func='sel_text'),
Attr(name='description', selector='.tab-1', func='sel_text'),
Attr(name='category', selector='li.normal', func='sel_text')
])
])
makro = ScrapeModel(name='makro',
domain='https://www.makro.nl/',
num_getters=1,
phases=[product])
d = Dispatcher()
d.add_scraper(makro)
d.run()