if sys.argv[1] == 'start': options = [] common_option = __import__( COMMON_SETTING_FILE.format(project_name=sys.argv[2]).replace( '/', '.').replace('\\', '.')[:-3], globals(), locals(), ['option']).option setting_files = glob.glob( SETTING_FILE.format(project_name=sys.argv[2])) for setting_file in setting_files: common_option.update( __import__( setting_file.replace('/', '.').replace('\\', '.')[:-3], globals(), locals(), ['option']).option) options.append(common_option.copy()) if len(options) > 1: thread_list = [ Sthread(thread_id=i, name=options[i]['site_name'], option=options[i]) for i in range(len(options)) ] for thread in thread_list: thread.start() else: spider = Spider(options[0]) spider.run()
def run(self): #getattr(globals()['class_name'](), 'function_name')('args') print('-|' * 5 + 'Starting ' + self.name + ' Thread ID:: ' + str(self.thread_id)) spider = Spider(self.option) spider.run()