def update_javlib_cf_cookies():
req_data = json.loads(request.get_data() or '{}')
update_dict = json.loads(req_data['update_dict'])
status = JavLibraryScraper.update_local_cookies(update_dict)
return jsonify({'status': status})
def search_by_car(car: str, **kwargs):
jav_obj = JavLibraryScraper({'car': car}).scrape_jav()
db_conn = JavManagerDB()
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
# use the full image (image key) instead of img (much smaller)
jav_obj['img'] = jav_obj.get('image', '')
return [jav_obj], 1
def cloudflare_get(url, cookies={}, proxies=None):
retry = 6
from JavHelper.core.javlibrary import JavLibraryScraper
while retry > 0:
try:
cookies.update(JavLibraryScraper.load_local_cookies()
) # update cloudflare cookies when updating
res = cloudscraper.create_scraper().get(url,
cookies=cookies,
proxies=proxies)
#print(res.text)
return res
#except cloudscraper.exceptions.CloudflareIUAMError:
except Exception as e:
print(f'cloudflare get failed on {e}, retrying {url}')
retry = retry - 1
sleep(5)
raise Exception(f'cloudflare get {url} failed')
def db_stat_lookup(page_num):
# handle db stat 0 look up
db_conn = JavManagerDB()
jav_objs, max_page = db_conn.query_on_filter({'stat': 0},
page=int(page_num))
# need additional info
for jav_obj in jav_objs:
if jav_obj['stat'] != 0:
print('0 stat detected, reindex and redo jav obj lookup')
db_conn.rebuild_index()
return db_stat_lookup(page_num)
if not jav_obj.get('title', None):
_full_info = JavLibraryScraper({
'car': jav_obj['car']
}).scrape_jav()
jav_obj.update(_full_info)
db_conn.upcreate_jav(jav_obj)
jav_obj.setdefault('img', jav_obj.get('image',
'')) # force img key to exist
return jav_objs, max_page
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if set_type == 'personal_wanted':
db_conn = JavManagerDB()
jav_objs, max_page = db_conn.query_on_filter({'stat': 0},
page=int(page_num))
# need additional info
for jav_obj in jav_objs:
if jav_obj['stat'] != 0:
db_conn.rebuild_index()
raise Exception(
'index is not up-to-date and it has been rebuild')
if not jav_obj.get('title', None):
_full_info = JavLibraryScraper({
'car': jav_obj['car']
}).scrape_jav()
jav_obj.update(_full_info)
db_conn.upcreate_jav(jav_obj)
jav_obj.setdefault('img',
jav_obj.get('image',
'')) # force img key to exist
# don't need extra db operations
return jsonify(
{'success': {
'jav_objs': jav_objs,
'max_page': max_page
}})
elif not search_string:
# parse set without search string
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify(
{'error': f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num)
else:
# search by supplied string
search_map = {
'番号': {
'function': search_by_car,
'params': {
'car': search_string
}
},
'女优': {
'function': search_for_actress,
'params': {
'javlib_actress_code': search_string,
'page_num': page_num
}
},
'分类': {
'function': javlib_set_page,
'params': {
'page_template':
'vl_genre.php?&mode=&g={url_parameter}&page={page_num}',
'page_num': page_num,
'url_parameter': search_string
}
},
}
# verify set type
if set_type not in search_map:
return jsonify(
{'error': f'{set_type} is not a supported search type'}), 400
jav_objs, max_page = search_map[set_type]['function'](
**search_map[set_type]['params'])
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify(
{'success': {
'jav_objs': rt_jav_objs,
'max_page': max_page
}})
def read_javlib_cf_cookies():
return jsonify({
'javlib_cf_cookies':
JavLibraryScraper.load_local_cookies(
return_all=request.args.get('return_all', False))
}) # convert returned obj to dict format
