def new_pick_index_rescrape():
car = request.args.get('car')
source = request.args.get('source')
pick_index = request.args.get('pick_index')
if not car:
return jsonify({'error': 'cannot find car from request'}), 400
if not pick_index.isdigit():
return jsonify({'error': f'{pick_index} is not a valid index'}), 400
# incremental pick index jav obj must exists currently
db_conn = JavManagerDB()
db_jav_obj = dict(db_conn.get_by_pk(car))
db_jav_obj_old = deepcopy(db_jav_obj)
# verify sources
sources = return_default_config_string('jav_obj_priority').split(',')
if source not in sources:
raise Exception(f'{source} is not a valid source for pick index update')
try:
scraped_info = SOURCES_MAP[source]({'car': car}, pick_index=int(pick_index)).scrape_jav()
except JAVNotFoundException:
errors = (db_jav_obj.get('errors') or [])
errors.append(
'{} cannot be found in {}'.format(db_jav_obj['car'], source)
)
scraped_info = {'errors': errors}
print(scraped_info)
db_jav_obj.update(scraped_info)
# also save it separate key
db_jav_obj[source] = scraped_info
if db_jav_obj_old != db_jav_obj:
db_conn.upcreate_jav(db_jav_obj)
return jsonify({'success': db_jav_obj})
def handle_jav_download(self, car: str, magnet: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except DoesNotExist:
jav_obj = {'car': car}
try:
# create download using magnet link
created_task = self.post_magnet_to_oof(magnet)
# get task detail from list page
search_hash = created_task['info_hash']
task_detail = self.get_task_detail_from_hash(search_hash)
# filter out unwanted files
download_files = self.filter_task_details(task_detail)
if not download_files:
raise Exception(f'there is no download file found in 115 task')
for download_file in download_files:
self.download_aria_on_pcode(download_file['cid'],
download_file['pickup_code'])
# if everything went well, update stat
jav_obj['stat'] = 4
db_conn.upcreate_jav(jav_obj)
return jav_obj
except Exception as e:
return {'error': f'download {car} failed due to {e}'}
def handle_jav_download(self, car: str, magnet: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except DoesNotExist:
jav_obj = {'car': car}
retry_num = 0
e = None
# create download using magnet link
try:
created_task = self.post_magnet_to_oof(magnet)
if created_task.get('errcode') == 10008:
return {
'error':
self.translate_map['oof_magnet_exists'].format(car=car)
}
except Exception as create_magnet_e:
return {
'error':
self.translate_map['oof_fail_magnet'].format(
car=car, create_magnet_e=create_magnet_e)
}
while retry_num < 3:
try:
# get task detail from list page
search_hash = created_task['info_hash']
task_detail = self.get_task_detail_from_hash(search_hash)
# filter out unwanted files
download_files = self.filter_task_details(task_detail)
if not download_files:
raise Exception(self.translate_map['oof_no_file'])
break
except Exception as _e:
retry_num += 1
sleep(5)
print(f'current error: {_e}, retrying')
e = _e
# send download info to aria2
try:
for download_file in download_files:
self.download_aria_on_pcode(download_file['cid'],
download_file['pickup_code'])
# if everything went well, update stat
jav_obj['stat'] = 4
db_conn.upcreate_jav(jav_obj)
return jav_obj
except Exception as _e:
print_exc()
e = _e
return {
'error':
self.translate_map['oof_general_failure'].format(
car=car, retry_num=retry_num, e=e)
}
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if not search_string:
# parse set without search string
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify({'error': f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javdb_set_page(SET_TYPE_MAP[set_type], page_num)
else:
# search by supplied string
search_map = {
'番号': {'function': search_by_car, 'params': {'car': search_string}},
'女优': {'function': search_for_actress, 'params': {
'javlib_actress_code': search_string, 'page_num': page_num}},
'分类': {'function': javdb_set_page, 'params':
{'page_template': 'tags?c6={url_parameter}&page={page_num}',
'page_num': page_num, 'url_parameter': search_string}
},
'系列': {'function': javdb_set_page, 'params':
{'page_template': 'series/{url_parameter}?page={page_num}',
'page_num': page_num, 'url_parameter': search_string}
}
}
# verify set type
if set_type not in search_map:
return jsonify({'error': BackendTranslation()['no_support_set_search'].format(set_type)}), 400
jav_objs, max_page = search_map[set_type]['function'](**search_map[set_type]['params'])
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(
dict(
db_conn.get_by_pk(str(jav_obj.get('car')))
)
)
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify({'success': {'jav_objs': rt_jav_objs, 'max_page': max_page}})
def get_local_car():
car = request.args.get('car')
db_conn = JavManagerDB()
if db_conn.pk_exist(car):
return jsonify({'success': dict(db_conn.get_by_pk(car))})
else:
return jsonify(
{'error':
f'{car} does not exist locally, car is case sensitive'}), 400
def need_ikoa_credit(car: str):
try:
db = JavManagerDB()
need = db.get_by_pk(car.upper()).get('need_ikoa_credit', '0') == "1"
print(f'need ikoa credit: {need}')
return need
except DoesNotExist as e:
# for any other error we return False
return False
except Exception as e:
# for any other error we return False
return False
def search_by_car(car: str, **kwargs):
jav_obj = JavLibraryScraper({'car': car}).scrape_jav()
db_conn = JavManagerDB()
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
# use the full image (image key) instead of img (much smaller)
jav_obj['img'] = jav_obj.get('image', '')
return [jav_obj], 1
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify({'error':
f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num)
db_conn = JavManagerDB()
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
return jsonify({'success': {'jav_objs': jav_objs, 'max_page': max_page}})
def search_for_actress(javlib_actress_code: str, page_num=1):
search_url = 'star/{url_parameter}/{page_num}'
db_conn = JavManagerDB()
# get actress first page
jav_objs, max_page = javbus_set_page(search_url,
page_num=page_num,
url_parameter=javlib_actress_code)
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jav_objs, max_page
def update_db_jav():
req_data = json.loads(request.get_data() or '{}')
jav_pk = req_data.get('pk')
update_data = req_data.get('data')
if not jav_pk:
return jsonify({'error': 'no pk found in posted json'}), 400
db_conn = JavManagerDB()
try:
current_jav_obj = dict(db_conn.get_by_pk(jav_pk))
except DoesNotExist:
current_jav_obj = {'car': jav_pk}
current_jav_obj.update(update_data)
db_conn.upcreate_jav(current_jav_obj)
return jsonify({'success': dict(current_jav_obj)})
def search_for_actress(javlib_actress_code: str, page_num=1):
"""
This only support javlibrary actress code
"""
search_url = 'vl_star.php?&mode=&s={url_parameter}&page={page_num}'
db_conn = JavManagerDB()
# get actress first page
jav_objs, max_page = javlib_set_page(search_url,
page_num=page_num,
url_parameter=javlib_actress_code)
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jav_objs, max_page
def find_images():
car = request.args.get('car')
sources = request.args.get('sources')
if not car:
return jsonify({'error': 'cannot find car from request'}), 400
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except DoesNotExist:
jav_obj = {'car': car}
# verify sources
if not sources:
sources = return_default_config_string('jav_obj_priority').split(',')
else:
sources = str(sources).split(',')
res = parse_single_jav({'car': car}, sources)
if res != jav_obj:
jav_obj.update(res)
db_conn.upcreate_jav(jav_obj)
return jsonify({'success': jav_obj})
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if set_type == 'personal_wanted':
db_conn = JavManagerDB()
jav_objs, max_page = db_conn.query_on_filter({'stat': 0},
page=int(page_num))
# need additional info
for jav_obj in jav_objs:
if jav_obj['stat'] != 0:
db_conn.rebuild_index()
raise Exception(
'index is not up-to-date and it has been rebuild')
if not jav_obj.get('title', None):
_full_info = JavLibraryScraper({
'car': jav_obj['car']
}).scrape_jav()
jav_obj.update(_full_info)
db_conn.upcreate_jav(jav_obj)
jav_obj.setdefault('img',
jav_obj.get('image',
'')) # force img key to exist
# don't need extra db operations
return jsonify(
{'success': {
'jav_objs': jav_objs,
'max_page': max_page
}})
elif not search_string:
# parse set without search string
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify(
{'error': f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num)
else:
# search by supplied string
search_map = {
'番号': {
'function': search_by_car,
'params': {
'car': search_string
}
},
'女优': {
'function': search_for_actress,
'params': {
'javlib_actress_code': search_string,
'page_num': page_num
}
},
'分类': {
'function': javlib_set_page,
'params': {
'page_template':
'vl_genre.php?&mode=&g={url_parameter}&page={page_num}',
'page_num': page_num,
'url_parameter': search_string
}
},
}
# verify set type
if set_type not in search_map:
return jsonify(
{'error': f'{set_type} is not a supported search type'}), 400
jav_objs, max_page = search_map[set_type]['function'](
**search_map[set_type]['params'])
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify(
{'success': {
'jav_objs': rt_jav_objs,
'max_page': max_page
}})
