def db_stat_lookup(page_num): # handle db stat 0 look up db_conn = JavManagerDB() jav_objs, max_page = db_conn.query_on_filter({'stat': 0}, page=int(page_num)) # need additional info for jav_obj in jav_objs: if jav_obj['stat'] != 0: print('0 stat detected, reindex and redo jav obj lookup') db_conn.rebuild_index() return db_stat_lookup(page_num) if not jav_obj.get('title', None): _full_info = JavLibraryScraper({ 'car': jav_obj['car'] }).scrape_jav() jav_obj.update(_full_info) db_conn.upcreate_jav(jav_obj) jav_obj.setdefault('img', jav_obj.get('image', '')) # force img key to exist return jav_objs, max_page
def get_set_javs(): set_type = request.args.get('set_type') page_num = request.args.get('page_num', 1) # optional search string search_string = request.args.get('search_string', '') _dedupe_car_list = [] rt_jav_objs = [] if set_type == 'personal_wanted': db_conn = JavManagerDB() jav_objs, max_page = db_conn.query_on_filter({'stat': 0}, page=int(page_num)) # need additional info for jav_obj in jav_objs: if jav_obj['stat'] != 0: db_conn.rebuild_index() raise Exception( 'index is not up-to-date and it has been rebuild') if not jav_obj.get('title', None): _full_info = JavLibraryScraper({ 'car': jav_obj['car'] }).scrape_jav() jav_obj.update(_full_info) db_conn.upcreate_jav(jav_obj) jav_obj.setdefault('img', jav_obj.get('image', '')) # force img key to exist # don't need extra db operations return jsonify( {'success': { 'jav_objs': jav_objs, 'max_page': max_page }}) elif not search_string: # parse set without search string # verify set type if set_type not in SET_TYPE_MAP: return jsonify( {'error': f'{set_type} is not a supported set type'}), 400 jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num) else: # search by supplied string search_map = { '番号': { 'function': search_by_car, 'params': { 'car': search_string } }, '女优': { 'function': search_for_actress, 'params': { 'javlib_actress_code': search_string, 'page_num': page_num } }, '分类': { 'function': javlib_set_page, 'params': { 'page_template': 'vl_genre.php?&mode=&g={url_parameter}&page={page_num}', 'page_num': page_num, 'url_parameter': search_string } }, } # verify set type if set_type not in search_map: return jsonify( {'error': f'{set_type} is not a supported search type'}), 400 jav_objs, max_page = search_map[set_type]['function']( **search_map[set_type]['params']) for jav_obj in jav_objs: if jav_obj['car'] not in _dedupe_car_list: _dedupe_car_list.append(jav_obj['car']) rt_jav_objs.append(jav_obj) # looooop through DB db_conn = JavManagerDB() for jav_obj in rt_jav_objs: if db_conn.pk_exist(str(jav_obj.get('car'))): jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car'))))) else: jav_obj['stat'] = 2 db_conn.upcreate_jav(jav_obj) return jsonify( {'success': { 'jav_objs': rt_jav_objs, 'max_page': max_page }})