def login(self):
"""Login LINE."""
self.driver.get(f'chrome-extension://{UID}/index.html')
sleep(0.5)
# browser may confirm leaving of the current page
try:
self.driver.switch_to.alert.accept()
except NoAlertPresentException:
pass
sleep(0.5)
# get email and password fields
email_field = self.driver.find_element_by_id('line_login_email')
pass_field = self.driver.find_element_by_id('line_login_pwd')
# clear fields
email_field.clear()
pass_field.clear()
# send keys
email_field.send_keys(EMAIL)
pass_field.send_keys(PASSWORD)
sleep(0.5)
self.driver.find_element_by_id('login_btn').click()
sleep(1.5)
while True:
try:
verify_code = (self.driver.find_element_by_css_selector(
'#login_content div.mdCMN01Code').text)
except NoSuchElementException:
logger.debug('no verify code needed.')
break
if verify_code:
logger.debug(f'input verify code: ({verify_code})')
else:
logger.info('Loggged in successfully!')
break
sleep(1)
def process_item(self, item, spider):
"""Save item to db."""
try:
self.db[self.collection_name].insert_one(dict(item))
except Exception as e:
logger.debug(str(e))
return item
def __init__(self, command_executor=None, reuse_session_id=None):
"""Add LINE chrome extension."""
info = None
if isfile(SESSION_CACHE_PATH):
with open(SESSION_CACHE_PATH) as f:
info = json.load(f)
chrome_options = Options()
chrome_options.add_extension(EXTENSTION_PATH)
if command_executor:
self.driver = RemotePatch(
command_executor=command_executor,
desired_capabilities=chrome_options.to_capabilities(),
reuse_session_id=reuse_session_id,
)
else:
if info:
self.driver = RemotePatch(
desired_capabilities=chrome_options.to_capabilities(),
**info)
logger.debug(f'resue: {info}')
else:
self.driver = RemotePatch(
command_executor=COMMAND_EXECUTOR,
desired_capabilities=chrome_options.to_capabilities(),
)
try:
# test if session still alive
logger.info(f'current url: {self.driver.current_url}')
except WebDriverException as e:
if 'No active session with ID' in e.args[0]:
logger.warn('session seems dead, create a new one...')
os.remove(SESSION_CACHE_PATH)
self.__init__()
def execute(self, command, params=None):
"""Patch for `webdriver.Remote.execute`."""
if command == 'newSession' and self.reuse_session_id:
logger.debug(f'reuse session: {self.reuse_session_id}')
return {'success': 0, 'value': None, 'sessionId': self.session_id}
else:
return super().execute(command, params)
def cache_session_info(self):
"""Cache session info."""
info = {
'command_executor': self.driver.command_executor._url,
'reuse_session_id': self.driver.session_id,
}
with open(SESSION_CACHE_PATH, 'w') as f:
json.dump(info, f)
logger.debug(f'dump session cache: {info}')
def parse_detail(self, response):
"""Parse detail house info fields."""
try:
city = response.meta['city']
house_id = response.meta['house_id']
detail_dict = dict([
tuple(''.join(el.split()).split(':'))
for el in response.css('.detailInfo li::text').getall()
])
house_type = detail_dict.get('型態', None)
house_recent = detail_dict.get('現況', None)
phone_number = response.css(
'.dialPhoneNum::attr(data-value)').get()
renter = response.meta['renter']
renter_sex = 0 if "先生" in renter else 1
renter_type = response.meta['renter_type']
rent_condition = response.css(
'.labelList em::attr(title)').getall()
if '男生' in rent_condition:
sex_limit = 0
elif '女生' in rent_condition:
sex_limit = 1
else:
sex_limit = 2
yield items.Rent591Item(house_id=house_id,
house_type=house_type,
house_recent=house_recent,
city=city,
renter=renter,
renter_type=renter_type,
renter_sex=renter_sex,
phone_number=phone_number,
sex_limit=sex_limit,
date_time=datetime.now())
except Exception as e:
self.except_list.append({
'house_id': response.meta['house_id'],
'message': str(e)
})
finally:
logger.debug(self.except_list)
def send_img(self, uri):
"""Send image.
Args:
- uri: image path (file://) or url (http://)
"""
logger.debug('opening new tab')
self.driver.execute_script(f'window.open("{uri}")')
sleep(0.5)
logger.debug('switching to newly opened tab')
self.driver.switch_to.window(self.driver.window_handles[-1])
sleep(0.5)
logger.debug('copying image')
(self.driver.find_element_by_tag_name('body').send_keys(
Keys.CONTROL, 'c'))
sleep(5)
logger.debug('closing tab')
self.driver.close()
sleep(0.5)
logger.debug('switching back to the main tab')
self.driver.switch_to.window(self.driver.window_handles[0])
sleep(1)
logger.debug('locating chat box')
ele = self.driver.find_element_by_id('_chat_room_input')
sleep(0.5)
logger.debug('clicking chat box')
ele.click()
sleep(1)
logger.debug('pasting image to chat box')
ele.send_keys(Keys.CONTROL, 'v')
sleep(5)
logger.debug('submitting')
ele.send_keys(Keys.RETURN)
logger.debug('done!')
# randomly click at any place to loose focus on the input box
self.driver.find_element_by_tag_name('body').click()
