if sys.argv[1] == 'start':
options = []
common_option = __import__(
COMMON_SETTING_FILE.format(project_name=sys.argv[2]).replace(
'/', '.').replace('\\', '.')[:-3], globals(), locals(),
['option']).option
setting_files = glob.glob(
SETTING_FILE.format(project_name=sys.argv[2]))
for setting_file in setting_files:
common_option.update(
__import__(
setting_file.replace('/', '.').replace('\\', '.')[:-3],
globals(), locals(), ['option']).option)
options.append(common_option.copy())
if len(options) > 1:
thread_list = [
Sthread(thread_id=i,
name=options[i]['site_name'],
option=options[i]) for i in range(len(options))
]
for thread in thread_list:
thread.start()
else:
spider = Spider(options[0])
spider.run()
def run(self):
#getattr(globals()['class_name'](), 'function_name')('args')
print('-|' * 5 + 'Starting ' + self.name + ' Thread ID:: ' +
str(self.thread_id))
spider = Spider(self.option)
spider.run()
