def update_javlib_cf_cookies(): req_data = json.loads(request.get_data() or '{}') update_dict = json.loads(req_data['update_dict']) status = JavLibraryScraper.update_local_cookies(update_dict) return jsonify({'status': status})
def search_by_car(car: str, **kwargs): jav_obj = JavLibraryScraper({'car': car}).scrape_jav() db_conn = JavManagerDB() if db_conn.pk_exist(str(jav_obj.get('car'))): jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car'))))) else: jav_obj['stat'] = 2 db_conn.upcreate_jav(jav_obj) # use the full image (image key) instead of img (much smaller) jav_obj['img'] = jav_obj.get('image', '') return [jav_obj], 1
def cloudflare_get(url, cookies={}, proxies=None): retry = 6 from JavHelper.core.javlibrary import JavLibraryScraper while retry > 0: try: cookies.update(JavLibraryScraper.load_local_cookies() ) # update cloudflare cookies when updating res = cloudscraper.create_scraper().get(url, cookies=cookies, proxies=proxies) #print(res.text) return res #except cloudscraper.exceptions.CloudflareIUAMError: except Exception as e: print(f'cloudflare get failed on {e}, retrying {url}') retry = retry - 1 sleep(5) raise Exception(f'cloudflare get {url} failed')
def db_stat_lookup(page_num): # handle db stat 0 look up db_conn = JavManagerDB() jav_objs, max_page = db_conn.query_on_filter({'stat': 0}, page=int(page_num)) # need additional info for jav_obj in jav_objs: if jav_obj['stat'] != 0: print('0 stat detected, reindex and redo jav obj lookup') db_conn.rebuild_index() return db_stat_lookup(page_num) if not jav_obj.get('title', None): _full_info = JavLibraryScraper({ 'car': jav_obj['car'] }).scrape_jav() jav_obj.update(_full_info) db_conn.upcreate_jav(jav_obj) jav_obj.setdefault('img', jav_obj.get('image', '')) # force img key to exist return jav_objs, max_page
def get_set_javs(): set_type = request.args.get('set_type') page_num = request.args.get('page_num', 1) # optional search string search_string = request.args.get('search_string', '') _dedupe_car_list = [] rt_jav_objs = [] if set_type == 'personal_wanted': db_conn = JavManagerDB() jav_objs, max_page = db_conn.query_on_filter({'stat': 0}, page=int(page_num)) # need additional info for jav_obj in jav_objs: if jav_obj['stat'] != 0: db_conn.rebuild_index() raise Exception( 'index is not up-to-date and it has been rebuild') if not jav_obj.get('title', None): _full_info = JavLibraryScraper({ 'car': jav_obj['car'] }).scrape_jav() jav_obj.update(_full_info) db_conn.upcreate_jav(jav_obj) jav_obj.setdefault('img', jav_obj.get('image', '')) # force img key to exist # don't need extra db operations return jsonify( {'success': { 'jav_objs': jav_objs, 'max_page': max_page }}) elif not search_string: # parse set without search string # verify set type if set_type not in SET_TYPE_MAP: return jsonify( {'error': f'{set_type} is not a supported set type'}), 400 jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num) else: # search by supplied string search_map = { '番号': { 'function': search_by_car, 'params': { 'car': search_string } }, '女优': { 'function': search_for_actress, 'params': { 'javlib_actress_code': search_string, 'page_num': page_num } }, '分类': { 'function': javlib_set_page, 'params': { 'page_template': 'vl_genre.php?&mode=&g={url_parameter}&page={page_num}', 'page_num': page_num, 'url_parameter': search_string } }, } # verify set type if set_type not in search_map: return jsonify( {'error': f'{set_type} is not a supported search type'}), 400 jav_objs, max_page = search_map[set_type]['function']( **search_map[set_type]['params']) for jav_obj in jav_objs: if jav_obj['car'] not in _dedupe_car_list: _dedupe_car_list.append(jav_obj['car']) rt_jav_objs.append(jav_obj) # looooop through DB db_conn = JavManagerDB() for jav_obj in rt_jav_objs: if db_conn.pk_exist(str(jav_obj.get('car'))): jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car'))))) else: jav_obj['stat'] = 2 db_conn.upcreate_jav(jav_obj) return jsonify( {'success': { 'jav_objs': rt_jav_objs, 'max_page': max_page }})
def read_javlib_cf_cookies(): return jsonify({ 'javlib_cf_cookies': JavLibraryScraper.load_local_cookies( return_all=request.args.get('return_all', False)) }) # convert returned obj to dict format