def run_scraper(self):
self.p.linker(self.p.scraper_dict, 'scraper')
scraper = Scraper(self.p.scraper)
processing = Processing(self.p.scraper)
scraper.run()
self.ProgressBar.setValue(25)
processing.pre_processor(scraper.scraper_df)
self.ProgressBar.setValue(50)
processing.spacy_processor(scraper.scraper_df)
self.ProgressBar.setValue(75)
self.connection = Connect(settings=None, mongo_cfg=self.p.scraper)
self.connection.update_from_df(scraper.scraper_df)
self.mongo_update()
self.ProgressBar.setValue(100)
action='store_true')
parser.add_argument('-sl',
'--set_local',
help='Use for setting local time.',
action='store_true')
args = parser.parse_args()
verbose = args.verbose
set_local = args.set_local
p = Parameters()
p.loader('save/params/default.params', 'params')
p.loader('dat/scraper.cfg', 'scraper')
scraper = Scraper(p.scraper)
scraper.run(verbose=verbose, set_local=set_local)
processor = Processing(p.scraper)
processor.pre_processor(scraper.scraper_df)
processor.spacy_processor(scraper.scraper_df, verbose=verbose)
connection = Connect(settings=p.params, mongo_cfg=p.scraper)
connection.update_from_df(scraper.scraper_df,
verbose=verbose,
set_local=set_local)
