def end(self):
if GlobalManager().debug:
print(self.__stack)
if self.params.get('key', None):
GlobalManager().set(self.params['key'],
copy.deepcopy(self.__stack))
ForManager().set_depth()
GlobalManager().loop_turn_on()
def end(self):
if self.params.get('xpath', None):
if not self.driver.find_elements_by_xpath(self.params['xpath']):
return GlobalManager().loop_turn_on()
if self.params.get('key', None):
if GlobalManager.get(self.params['key'], '_db_args'):
return GlobalManager().loop_turn_on()
return GlobalManager().loop_turn_off()
def start(self):
try:
ForManager().for_loop_start()
except Exception as e:
if GlobalManager().debug:
print(traceback.format_exc())
print(e)
if self.params.get('turn_on', True):
GlobalManager().loop_turn_off()
ForManager().destroy_for_stack()
def wrapper(*args, **kwargs):
start = timeit.default_timer()
__func = func(*args, **kwargs)
end = timeit.default_timer()
if GlobalManager().debug:
print('run time: {} s'.format(str(round(end - start, 4))))
return __func
def url_format(self):
global_type = self.params.get('global_type', None)
if GlobalManager().debug:
print('global_type：', global_type)
if global_type is None:
params = GlobalManager().get()
else:
params = GlobalManager().get(type_=global_type)
if self.params.get('param', None):
return self.params['param'].format(**params)
if self.params.get('url', None):
url = self.params['url'].format(**params)
return url
def start_block(self):
GlobalManager().if_turn_off()
if self.params.get(
'xpath', None) and self.driver.find_elements_by_xpath('xpath'):
GlobalManager().if_turn_on()
return True
if self.params.get(
'not_exist_xpath', None
) and not self.driver.find_elements_by_xpath('not_exist_xpath'):
GlobalManager().if_turn_on()
return True
if self.params.get('attr_xpath', None) and self.driver.find_element_by_xpath('attr_xpath')\
.get_attribute(self.params['attr']):
GlobalManager().if_turn_on()
return True
return False
def grab(self):
element = ForManager().get_element_for_stack()
if self.params.get('xpath', None):
element = element.find_element_by_xpath(self.params['xpath'])
if self.params.get('tag_name', None):
element = element.find_element_by_tag_name(self.params['tag_name'])
if GlobalManager().debug:
print(element.text)
self.__stack[self.params['key']] = screening_value(
element, self.params)
def loop_condition(self):
elements = []
if self.params.get('xpath', None):
elements = self.driver.find_elements_by_xpath(self.params['xpath'])
if self.params.get('value', None):
elements = self.params['value']
if self.params.get('storage', None):
elements = GlobalManager().get(self.params['storage'])
for element in elements:
yield element
def build(self, spider_type):
self.file_name = spider_type
if '{}.py'.format(self.file_name) not in self.hooks_name:
raise Exception('{} is not exist in hooks dir'.format(
self.file_name))
if self.__hook is None:
load_module = importlib.import_module('core.hooks.{}'.format(
self.file_name))
if GlobalManager().debug:
print(load_module)
self.__hook = getattr(load_module,
convert_big_hump(self.file_name))
return self.__hook()
def init(self, spider_type, task_name):
self.__spider_type = spider_type
self.__task_name = task_name
self.__container = Container()
self.set('app', self)
self.set('browser', Browser())
self.set('facade', Facade())
Facade().init(self)
self.set('component', ComponentManager())
self.set('for', ForManager())
self.set('global', GlobalManager())
self.set('hook', HookManager())
self.set('config', Config())
self.set('engine', Engine())
Engine().init(spider_type)
return self
def __init__(self):
self.file_name = None
self.hooks_name = os.listdir(Config().get_dir().hooks())
if GlobalManager().debug:
print('use hook: ', self.hooks_name)
def has_class_terminate(self):
element = self.driver.find_element_by_xpath(self.params['xpath'])
if self.params['class'] in element.get_attribute('class'):
return GlobalManager().loop_turn_off()
return GlobalManager().loop_turn_on()
def end_block():
GlobalManager().if_turn_off()
def for_loop_start(self):
if GlobalManager().debug:
print(self.for_stack)
element = next(self.for_stack[self.depth]['yield'])
self.set_element_for_stack(element)
def set_depth(self):
self.depth = GlobalManager().depth
def not_has_element_terminate(self):
element = self.driver.find_elements_by_xpath(self.params['xpath'])
if not element:
return GlobalManager().loop_turn_off()
return GlobalManager().loop_turn_on()
def enable_click_loop(self):
element = self.driver.find_element_by_xpath(self.params['xpath'])
if element.is_enabled():
return GlobalManager().loop_turn_on()
return GlobalManager().loop_turn_off()
def __init__(self, params):
super(For, self).__init__(params)
ForManager().set_depth()
if GlobalManager().component_type == 'start' and ForManager(
).get_yield_for_stack() is None:
ForManager().set_yield_for_stack(self.loop_condition())
