def find_element(self, selector, value):
method = getattr(self.driver, f'find_element_by_{selector}')
try:
element = method(value)
return element
except Exception as e:
logger.error("[%s][%s]find element error, %s", self.name,
self._step['name'], str(e))
return None
def upadte_item(self, item):
sql = self.pool.get_sql()
limit_dict = {'id': item.pop('source_id')}
code, msg = OriginSqlItem.update_item(sql,
TABLE['hotspot-source'],
value_dict=item,
limit_dict=limit_dict)
if not code:
logger.error("update status error, %s", msg)
sql.close()
def input(self, message):
selector: dict = self._step['selector']
for selector, value in selector.items():
element = self.find_element(selector, value)
if element:
element.send_keys(message)
return True
else:
logger.error("[%s][%s]input account element not found", self.name,
self._step['name'])
return False
def is_different_board(self, board_md5):
board_key = f's_{board_md5}'
try:
if self._redis.exists(board_key):
return False
self._redis.set(board_key, '')
self._redis.expire(board_key, DAY * 3)
return True
except Exception as e:
logger.error("redis error: %s", str(e))
return False
def is_different_link(self, board_id, link_md5):
link_key = f'b_{board_id}_{link_md5}'
try:
if self._redis.exists(link_key):
return False
self._redis.set(link_key, '')
self._redis.expire(link_key, DAY * 3)
return True
except Exception as e:
logger.error("redis error: %s", str(e))
return False
def save(self, table, item):
save_cmd = self.get_save_cmd(table, item)
sql = self.pool.get_sql()
try:
sql.cursor.execute(save_cmd)
sql.conn.commit()
except IntegrityError: # 重复
return False
except Exception as e:
logger.error("save error: %s", str(e))
return False
finally:
sql.close()
return True
def parse_wrapper(parser, response):
result = parse_method(parser, response)
if not isinstance(result, GeneratorType):
return logger.error("empty returned from parse")
try:
for item in result:
if isinstance(item, Request):
yield item
elif isinstance(item, tuple):
code, value = item
if code == Value.HOT_NUM:
parser.log_hot_num += value
elif isinstance(item, dict):
item.update(parser.item)
item['hot_md5'] = get_md5(item['url'])
if NewsRecorder.record(item):
parser.log_upload_num += 1
except Exception as e:
logger.info('list error: %s', str(e))
parser._abnormal = Abnormal.abnormal(
code=30, msg=Abnormal.ERROR_SOURCE_DOC[1])
logger.info("[%s][%s]get %s hot num in %s", parser.config_id,
parser.name, parser.log_hot_num, response.url)
parser._is_working = False
def get_step(self, step_method):
for step in self._steps:
if step['method'] == step_method:
return step
else:
logger.error("step not found by %s", step_method)