def insert_data(): first_name = input("please enter your first name") last_name = input("please enter your last name") mail = input("please enter your email address") notify_input = input( "please enter 1 if you want to get notification every time an item is uploaded or 0 for once a day" ) # notify_input = check_input(notify_input, int) user = User(first_name=first_name, last_name=last_name, email=mail, notify=notify_input) db.session.add(user) db.session.commit() print( "please enter the items you are looking for and the maximum price for each item" ) items_list = [] while "*" != input("press * when finished"): item_name = input("enter the item name") item_price = input("maximum price for " + item_name) item = Item(name=item_name, max_price=item_price) items_list.append(item) db.session.add(item) db.session.commit() search = Search(user_id=user.id, items=items_list) db.session.add(search) db.session.commit()
def api_stream_days(): stream_id = request.args.get('id', type=int) if not stream_id: abort(400) days_facet = Search().get_stream_days(stream_id) return jsonify( {'days': sorted(term['term'] for term in days_facet['terms'])})
def api_stream_records(): stream_id = request.args.get('id', type=int) if not stream_id: abort(400) date = request.args.get('date', type=str) if not date: abort(400) records = [] for result in Search().stream_records(stream_id, date=date).results: records.append({ 'title': result['title'], 'duration': result['duration'], 'ts': result['ts'], 'url': full_url_for('play_redirect', record_id=result['record_id']), }) return jsonify({'records': records})
def search(keyword): global driver global search try: if driver is None: driver = set_driver(IsHeadless) except Exception as error: error_logger('Seleniumドライバ設定エラー', error) return None try: url = 'https://ja.aliexpress.com/all-wholesale-products.html?spm=a2g0o.category_nav.0.0.300734f6uRXi28' search = Search(url, keyword) search.set_keyword(driver) search.set_currency(driver) search.get_category(driver) eel.message('検索完了、カテゴリを設定しました。絞り込みを必要に応じて設定してください。') eel.set_category_info(search.categories) eel.view_current_url(driver.current_url) except Exception as error: error_logger('検索エラー', error) return None
import logging import multiprocessing as mp from app import Retrieve from app import Search from app import Extract logging.basicConfig(level=logging.INFO) sr = Search() retr = Retrieve(useproxy=True, awsprofile='default') ex = Extract(awsprofile='default') # list = sr.incomplete(category='user', step='all', getitems=100000) # # def mp_retrieve(user): # retr.retrieve_user(user) # # pool = mp.Pool(processes=10) # # pool.map(mp_retrieve, list) # list = sr.incomplete(category='picture') # # logging.info(list) # logging.info(len(list)) # # for post in list: # # for post in list[0:1]: # try: # retr.retrieve_picture(post)
import eel import settings from app import Search, Item from scraping import set_driver from datetime import datetime as dt from logging import getLogger, FileHandler, StreamHandler, Formatter, DEBUG import pandas as pd app_name = "web" end_point = "index.html" size = (800, 750) url = 'https://www.buyma.com/r/' search_goods = Search(url) item = Item('') IsHeadless = False driver = None logger = getLogger(__name__) fomatterSetting = Formatter( '[%(asctime)s] %(name)s %(threadName)s %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S') handler = FileHandler('logger.log') # handler = StreamHandler() # handler = NullHandler() handler.setLevel(DEBUG) logger.setLevel(DEBUG) handler.setFormatter(fomatterSetting) logger.addHandler(handler) logger.propagate = False
import eel import settings from app import Search, Item from scraping import set_driver, parse_html from datetime import datetime as dt from logging import getLogger, FileHandler, StreamHandler, Formatter, DEBUG import pandas as pd app_name = "web" end_point = "index.html" size = (850, 850) IsHeadless = False driver = None search = Search('', '') items = [] logger = getLogger(__name__) fomatterSetting = Formatter( '[%(asctime)s] %(name)s %(threadName)s %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S') # handler = FileHandler('logger.log') handler = StreamHandler() # handler = NullHandler() handler.setLevel(DEBUG) logger.setLevel(DEBUG) handler.setFormatter(fomatterSetting) logger.addHandler(handler) logger.propagate = False # @eel.expose