def run_scraper(self): self.p.linker(self.p.scraper_dict, 'scraper') scraper = Scraper(self.p.scraper) processing = Processing(self.p.scraper) scraper.run() self.ProgressBar.setValue(25) processing.pre_processor(scraper.scraper_df) self.ProgressBar.setValue(50) processing.spacy_processor(scraper.scraper_df) self.ProgressBar.setValue(75) self.connection = Connect(settings=None, mongo_cfg=self.p.scraper) self.connection.update_from_df(scraper.scraper_df) self.mongo_update() self.ProgressBar.setValue(100)
action='store_true') parser.add_argument('-sl', '--set_local', help='Use for setting local time.', action='store_true') args = parser.parse_args() verbose = args.verbose set_local = args.set_local p = Parameters() p.loader('save/params/default.params', 'params') p.loader('dat/scraper.cfg', 'scraper') scraper = Scraper(p.scraper) scraper.run(verbose=verbose, set_local=set_local) processor = Processing(p.scraper) processor.pre_processor(scraper.scraper_df) processor.spacy_processor(scraper.scraper_df, verbose=verbose) connection = Connect(settings=p.params, mongo_cfg=p.scraper) connection.update_from_df(scraper.scraper_df, verbose=verbose, set_local=set_local)