def get_fetch(log=False): settings = Settings() settings.set('LOG_ENABLED', log) crawler_process = CrawlerProcess(settings) crawler = crawler_process.create_crawler() crawler_process.start_crawling() t = Thread(target=crawler_process.start_reactor) t.daemon = True t.start() shell = Shell(crawler) shell.code = 'adsf' import threading lock = threading.Lock() def fetch(url_or_request): lock.acquire() try: shell.fetch(url_or_request) response = shell.vars.get('response') return response finally: lock.release() return fetch
def shell(argv): """ Open a url in the scrapy shell """ parser = argparse.ArgumentParser('ozzy shell', description=shell.__doc__) parser.add_argument('url', help="URL to open in a shell") args = parser.parse_args(argv) crawler_process = CrawlerProcess(load_settings()) crawler = crawler_process.create_crawler() crawler_process.start_crawling() thread = Thread(target=crawler_process.start_reactor) thread.daemon = True thread.start() sh = Shell(crawler) sh.start(url=args.url)