def input_secret_word():
log.info("need secret word.")
e_q = driver.find_element(By.XPATH,
'//dt[text()="質問"]/following-sibling::dd')
log.debug("q=%s %s" % (e_q.tag_name, e_q.text))
log.info("waiting word input ...")
print(e_q.text)
word = input('Enter Secret word :')
e_word = driver.find_element(By.NAME, 'aikotoba')
e_word.send_keys(word)
e_word_link = driver.find_element(By.XPATH, '//a[text()="次へ"]')
helper.ss(name='before-submit-word')
e_word_link.click()
helper.ss(name='after-submit-word')
# 処理開始
try:
helper.user_agent='Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko'
driver = helper.start_browser()
helper.set_download(outdir)
driver.set_window_size(1920,1080)
# SMBC VISA TOP
log.info("getting SMBC Visa top page")
log.debug('window size = %s' % driver.get_window_size() )
driver.get('https://www.smbc-card.com/mem/index.jsp')
time.sleep(10)
page_width = driver.execute_script('return document.body.scrollWidth')
page_height = driver.execute_script('return document.body.scrollHeight')
log.debug('page size width=%d height=%d' % (page_width,page_height))
driver.set_window_size(page_width, page_height)
helper.ss(name='smbcvisa_top')
#log.debug('cookies = {}'.format(driver.get_cookies()))
e_user = driver.find_element_by_name('userid')
log.debug('e_user : {} visible={}'.format(e_user.get_attribute('outerHTML'),e_user.is_displayed()))
e_capy = driver.find_element_by_class_name('capy-captcha')
log.debug('capy element : tag={} id={} visible={}'.format(e_capy.tag_name,e_capy.get_attribute('id'),e_capy.is_displayed()))
capy_id = re.sub(r'capy$',"",e_capy.get_attribute('id'))
log.info('capy id = '+capy_id)
e_image_area = driver.find_element_by_xpath('//div[@id="'+capy_id+'image-area"]')
e_image = e_image_area.find_element_by_name(capy_id+'image')
# log.debug('image-area image={}'.format(e_image.get_attribute('outerHTML')))
e_pieces = driver.find_elements_by_xpath('//div[@id="'+capy_id+'pieces"]/div')
for i,e in enumerate(e_pieces):
# log.debug('piece[{}] : {} visible={} location={}'.format(i, e.get_attribute('outerHTML'),e.is_displayed(),e.location))
log.debug('piece[%d] : visible=%s location=%s outerHTML=%s' % (i, e.is_displayed(),e.location,e.get_attribute('outerHTML')))
def crawl(self, request):
print("Hello,rakuten")
# ログ設定
log = self.log
log.setLevel(logging.DEBUG)
# h = logging.handlers.TimedRotatingFileHandler('{}/{}.log'.format(LOGDIR,self.ap_name),'D',2,13)
# h.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
# log.addHandler(h)
#h = logging.handlers.SysLogHandler(address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_LOCAL1)
h = logging.StreamHandler(sys.stdout)
log.addHandler(h)
log.info("start")
return
OUTDIR_SS='./file/ss/'
OUTDIR='./file/'
LOGDIR='./log/'
# 引数解析
usage = 'Usage: python {} USER_ID PASSWORD [--outdir <dir>] [--help]'.format(__file__)
parser = ArgumentParser(usage=usage)
parser.add_argument('user_id', type=str, help='USER ID')
parser.add_argument('passwd', type=str, help='PASSWORD')
parser.add_argument('-o', '--outdir', type=str, help='file output directory', default=OUTDIR)
args = parser.parse_args()
user_id = args.user_id
passwd = args.passwd
outdir = args.outdir
helper.outdir_ss = OUTDIR_SS
helper.is_save_html_with_ss = True
log.info("outdir_ss={}".format(helper.outdir_ss))
# 処理開始
try:
# ブラウザを起動
driver = helper.start_browser()
driver.set_page_load_timeout(30)
helper.set_download(outdir)
# 楽天ログイン画面
log.info("getting login page")
driver.get('https://www.rakuten-card.co.jp/e-navi/')
helper.ss(name='login')
# ログイン
log.info("logging in to the site...")
e_user = driver.find_element_by_id('u')
e_password = driver.find_element_by_id('p')
e_user.send_keys(user_id)
e_password.send_keys(passwd)
e_button = driver.find_element_by_id('loginButton')
helper.ss(name='before-login')
e_button.click()
helper.ss(name='top')
# ご利用明細へのリンクを探す
log.info("navigate to bill-detail..")
e_link = driver.find_element_by_xpath('//ul[@class="rce-u-list-plain"]//a[text()="ご利用明細"]')
log.debug('link for detail : tag={} href={} visible={}'.format(e_link.tag_name, e_link.get_attribute('href'),e_link.is_displayed()))
#helper.ss(name='detail_link')
# ご利用明細
e_link.click()
helper.ss(name='detail')
# 確定かどうか判定
#e_bill = driver.find_element_by_xpath('//div[@class="stmt-bill-info-amount__main"]/span[contains(@class,"mark")]')
e_bill = driver.find_element_by_xpath('//div[@class="stmt-about-payment__money__main"]')
log.debug('bill-text:{}'.format(e_bill.text))
# 未確定なら支払い予定額を保存
if '未確定' in e_bill.text:
self.write_future_expence(driver, outdir)
log.info("end")
exit()
# CSVダウンロードのリンクを探す
log.info("downloading csv..")
e_csv_link = driver.find_element_by_xpath('//a[contains(.,"CSV")]')
log.debug('link for csv : tag={} href={}'.format(e_csv_link.tag_name, e_csv_link.get_attribute('href')))
# download csv
driver.get(e_csv_link.get_attribute('href'))
helper.ss(name='csv_downloaded')
# 次月が押せたら次月を押す
log.info("navigate to next month ..")
e_next_link = driver.find_element_by_xpath('//a[text()="次月"]')
log.debug('link for next month : tag={} href={} visible={}'.format(e_next_link.tag_name, e_next_link.get_attribute('href'),e_next_link.is_displayed()))
e_next_link.click()
helper.ss(name='next_month')
# 支払い予定金額出力
self.write_future_expence(driver, outdir)
finally:
if ( driver is not None ):
driver.quit()
log.info("WebDriver Quit")
log.info("end")
return
log.info("outfile=%s" % outfile)
# 処理開始
try:
driver = helper.start_browser()
helper.outdir_ss = os.path.join(OUTDIR_SS, account)
log.info("outdir_ss={}".format(helper.outdir_ss))
helper.set_download(outdir)
helper.is_save_html_with_ss = True
driver.set_page_load_timeout(60)
# ログイン画面
log.info("getting SMBC login page")
driver.get('https://direct.smbc.co.jp/aib/aibgsjsw5001.jsp')
time.sleep(1)
helper.ss(name='smbc_login')
# ############# system maintainance check
try:
e_sorry = driver.find_element(By.CSS_SELECTOR, 'div#sorry')
log.error("システムメンテナンス中")
log.error("error end")
sys.exit(9)
except selenium.common.exceptions.NoSuchElementException:
pass
# ログイン
log.info("logging in to the site...")
e_tab = driver.find_element(By.ID, 'tab-switchB02')
e_tab.click()
driver = None
# 処理開始
try:
# helper.user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
helper.user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
driver = helper.start_browser()
helper.outdir_ss = OUTDIR_SS
helper.set_download(outdir)
helper.is_save_html_with_ss = True
driver.set_page_load_timeout(-1) # max timeout
# AEON TOP
log.info("getting AEON top page")
driver.get('https://www.aeon.co.jp/')
helper.ss(name='aeon_top')
# ログイン画面
log.info("getting login page")
url = 'https://www.aeon.co.jp/app/'
log.info("login page url=" + url)
driver.get(url)
helper.ss(name='aeon_login')
login_url = driver.current_url
log.info('login url={}'.format(login_url))
# ログイン
log.info("logging in to the site...")
time.sleep(5)
# e_user = driver.find_element_by_name('netMemberId')
e_user = driver.find_element_by_id('username')
helper.outdir_ss = OUTDIR_SS
helper.is_save_html_with_ss = True
log.info("outdir_ss={}".format(helper.outdir_ss))
# 処理開始
try:
# ブラウザを起動
driver = helper.start_browser()
driver.set_page_load_timeout(30)
helper.set_download(outdir)
# 楽天ログイン画面
log.info("getting login page")
driver.get('https://www.rakuten-card.co.jp/e-navi/')
helper.ss(name='login')
# ログイン
log.info("logging in to the site...")
e_user = driver.find_element(By.ID, 'u')
e_password = driver.find_element(By.ID, 'p')
e_user.send_keys(user_id)
e_password.send_keys(passwd)
e_button = driver.find_element(By.ID, 'loginButton')
helper.ss(name='before-login')
e_button.click()
helper.ss(name='top')
# ご利用明細へのリンクを探す
log.info("navigate to bill-detail..")
e_link = driver.find_element(By.XPATH, '//ul[@class="rce-u-list-plain"]//a[text()="ご利用明細"]')
# 処理開始
try:
driver = helper.start_browser()
helper.outdir_ss = OUTDIR_SS
log.info("outdir_ss={}".format(helper.outdir_ss))
helper.set_download(outdir)
helper.is_save_html_with_ss = True
# ログイン画面
log.info("getting Yuucho login page")
driver.get(
'https://direct.jp-bank.japanpost.jp/tp1web/U010101WAK.do?link_id=ycDctLgn'
)
time.sleep(1)
helper.ss(name='yuucho_login')
# ログイン(お客様番号入力）
log.info("inputting user ids for login..")
e_user1 = driver.find_element(By.NAME, 'okyakusamaBangou1')
e_user2 = driver.find_element(By.NAME, 'okyakusamaBangou2')
e_user3 = driver.find_element(By.NAME, 'okyakusamaBangou3')
e_submit = driver.find_element(
By.XPATH, '//input[@type="button" and @name="U010103"]')
e_user1.send_keys(user1)
e_user2.send_keys(user2)
e_user3.send_keys(user3)
helper.ss(name='before-input-userID')
e_submit.click()
helper.ss(name='after-input-userID')