def __get_kindle_books(wish_list_url: str) -> dict:
headless_chrome = HeadlessChrome()
wish_list = WishList(url=wish_list_url, headless_chrome=headless_chrome)
book_url_list = wish_list.get_kindle_book_url_list()
kindle_books_list = wish_list.get_kindle_books(book_url_list)
headless_chrome.driver.close()
return kindle_books_list
def worker_scraping_wish_list(event, context):
# sqs への通信でエラーになった時余計な処理をしないために最初に取得する
sqs = boto3.client('sqs')
queue = sqs.get_queue_url(QueueName="worker_scraping_book")
logger.debug("sqs queue: %s", queue)
url_in_wish_list = []
headless_chrome = HeadlessChrome()
for record in event['Records']:
body = json.loads(record['body'])
logger.info("JSON body: %s", body)
wish_list_url = body['url']
logger.info("wish_list_url: %s", wish_list_url)
wish_list = WishList(url=wish_list_url,
headless_chrome=headless_chrome)
book_url_list = wish_list.get_kindle_book_url_list()
url_in_wish_list.extend(book_url_list)
headless_chrome.driver.close()
unique_url_list = list(set(url_in_wish_list))
for url in unique_url_list:
# SQSへJSONの送信
body = {"url": url}
response = sqs.send_message(QueueUrl=queue['QueueUrl'],
DelaySeconds=0,
MessageBody=(json.dumps(body)))
logger.info("sqs response: %s", response)
logger.info("url_in_wish_list: %s", unique_url_list)
return unique_url_list
def __init__(self, url, headless_chrome=None):
if headless_chrome is None:
self.headless_chrome = HeadlessChrome()
else:
self.headless_chrome = headless_chrome
html = get_full_page_html(self.headless_chrome.driver, url)
soup = BeautifulSoup(html, "html.parser")
self.soup = soup
self.kindle_book = KindleBook(headless_chrome)
def worker_scraping_book(event, context):
headless_chrome = HeadlessChrome()
kindle_book = KindleBook(headless_chrome=headless_chrome)
kindle_books = []
for record in event['Records']:
body = json.loads(record['body'])
logger.info("JSON body: %s", body)
kindle_book_url = body['url']
book_dict = kindle_book.get(url=kindle_book_url)
kindle_books.append(book_dict)
headless_chrome.driver.close()
logger.info("kindle_books: %s", kindle_books)
return kindle_books
def on_event(queue_item: dict) -> None:
headless_chrome = HeadlessChrome()
wish_list_url = queue_item['wish_list_url']
wish_list = WishList(url=wish_list_url, headless_chrome=headless_chrome)
book_url_list = wish_list.get_kindle_book_url_list()
with ThreadPoolExecutor(thread_name_prefix="thread") as executor:
futures = []
for book_url in book_url_list:
futures.append(executor.submit(invoke_lambda, book_url,
queue_item))
logger.info("submit end")
logger.info([f.result() for f in futures])
headless_chrome.driver.close()
def test_get_kindle_books(self):
event = {
'wish_list_url': 'http://amzn.asia/4jy4esM',
'expired': 1545440835,
'user_id': '92676047-1ce7-4080-bd44-10e3fc63a16d',
'description': 'マンガ',
'threshold': {
'discount_rate': 20,
'points': 20
}
}
try:
headless_chrome = HeadlessChrome()
get_kindle_books(event, headless_chrome)
headless_chrome.driver.close()
except:
self.fail()
def lambda_handler(event, context):
"""
Trigger: dynamodb insert event
monitor から item を抜き出し enqueue する
:param event:
:param context:
:return:
"""
headless_chrome = HeadlessChrome()
kindle_book = KindleBook(headless_chrome)
url = event["item_url"]
book = kindle_book.get(url=url)
res = {url: book}
headless_chrome.driver.close()
logger.info("res: %s", res)
notify(event, res)
def kindle_book():
headless_chrome = HeadlessChrome()
kindle_book = KindleBook(headless_chrome)
yield(kindle_book)
headless_chrome.driver.close()
def __init__(self, headless_chrome=None):
if headless_chrome is None:
self.headless_chrome = HeadlessChrome()
else:
self.headless_chrome = headless_chrome
def test_init():
hc = HeadlessChrome()
hc.driver.close()