def new_pick_index_rescrape():
car = request.args.get('car')
source = request.args.get('source')
pick_index = request.args.get('pick_index')
if not car:
return jsonify({'error': 'cannot find car from request'}), 400
if not pick_index.isdigit():
return jsonify({'error': f'{pick_index} is not a valid index'}), 400
# incremental pick index jav obj must exists currently
db_conn = JavManagerDB()
db_jav_obj = dict(db_conn.get_by_pk(car))
db_jav_obj_old = deepcopy(db_jav_obj)
# verify sources
sources = return_default_config_string('jav_obj_priority').split(',')
if source not in sources:
raise Exception(f'{source} is not a valid source for pick index update')
try:
scraped_info = SOURCES_MAP[source]({'car': car}, pick_index=int(pick_index)).scrape_jav()
except JAVNotFoundException:
errors = (db_jav_obj.get('errors') or [])
errors.append(
'{} cannot be found in {}'.format(db_jav_obj['car'], source)
)
scraped_info = {'errors': errors}
print(scraped_info)
db_jav_obj.update(scraped_info)
# also save it separate key
db_jav_obj[source] = scraped_info
if db_jav_obj_old != db_jav_obj:
db_conn.upcreate_jav(db_jav_obj)
return jsonify({'success': db_jav_obj})
def create_app():
# initialize local db and index
_db = JavManagerDB()
_db.create_indexes()
# create and configure the app
app = Flask(__name__, template_folder='templates')
cache.init_app(app)
app.register_blueprint(emby_actress)
app.register_blueprint(parse_jav)
app.register_blueprint(jav_browser)
app.register_blueprint(directory_scan)
app.register_blueprint(local_manager)
app.config['JSON_AS_ASCII'] = False
# a simple page that says hello
@app.route('/')
def hello():
return render_template('home.html')
@app.errorhandler(Exception)
def handle_exception(e):
# pass through HTTP errors
if isinstance(e, HTTPException):
return e
print_exc()
# now you're handling non-HTTP exceptions only
return jsonify({'errors': format_exc()}), 500
return app
def handle_jav_download(self, car: str, magnet: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except DoesNotExist:
jav_obj = {'car': car}
try:
# create download using magnet link
created_task = self.post_magnet_to_oof(magnet)
# get task detail from list page
search_hash = created_task['info_hash']
task_detail = self.get_task_detail_from_hash(search_hash)
# filter out unwanted files
download_files = self.filter_task_details(task_detail)
if not download_files:
raise Exception(f'there is no download file found in 115 task')
for download_file in download_files:
self.download_aria_on_pcode(download_file['cid'],
download_file['pickup_code'])
# if everything went well, update stat
jav_obj['stat'] = 4
db_conn.upcreate_jav(jav_obj)
return jav_obj
except Exception as e:
return {'error': f'download {car} failed due to {e}'}
def update_jav_dict():
# update db jav dict, also rewrite nfo and images
req_data = json.loads(request.get_data() or '{}')
update_dict = req_data['update_dict']
# update db
db_conn = JavManagerDB()
db_conn.upcreate_jav(update_dict)
file_writer = EmbyFileStructure(return_default_config_string('file_path'))
# file structure operations
try:
jav_obj = file_writer.create_folder_for_existing_jav(update_dict)
except KeyError as e:
_car = update_dict.get('car', 'Unknown')
update_dict.append(json.dumps({'log': f'error: {e}, skipping {_car}'}))
# write images
file_writer.write_images(jav_obj)
# write nfo
file_writer.write_nfo(jav_obj)
# move video file
jav_obj = file_writer.move_existing_file(jav_obj)
return jsonify({'success': jav_obj}) # post updated jav_obj back to UI
def partial_search():
search_string = request.args.get('search_string')
db_conn = JavManagerDB()
rt = db_conn.partial_search(search_string)
return jsonify({'success': [dict(x) for x in rt]})
def handle_jav_download(self, car: str, magnet: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except DoesNotExist:
jav_obj = {'car': car}
retry_num = 0
e = None
# create download using magnet link
try:
created_task = self.post_magnet_to_oof(magnet)
if created_task.get('errcode') == 10008:
return {
'error':
self.translate_map['oof_magnet_exists'].format(car=car)
}
except Exception as create_magnet_e:
return {
'error':
self.translate_map['oof_fail_magnet'].format(
car=car, create_magnet_e=create_magnet_e)
}
while retry_num < 3:
try:
# get task detail from list page
search_hash = created_task['info_hash']
task_detail = self.get_task_detail_from_hash(search_hash)
# filter out unwanted files
download_files = self.filter_task_details(task_detail)
if not download_files:
raise Exception(self.translate_map['oof_no_file'])
break
except Exception as _e:
retry_num += 1
sleep(5)
print(f'current error: {_e}, retrying')
e = _e
# send download info to aria2
try:
for download_file in download_files:
self.download_aria_on_pcode(download_file['cid'],
download_file['pickup_code'])
# if everything went well, update stat
jav_obj['stat'] = 4
db_conn.upcreate_jav(jav_obj)
return jav_obj
except Exception as _e:
print_exc()
e = _e
return {
'error':
self.translate_map['oof_general_failure'].format(
car=car, retry_num=retry_num, e=e)
}
def get_local_car():
car = request.args.get('car')
db_conn = JavManagerDB()
if db_conn.pk_exist(car):
return jsonify({'success': dict(db_conn.get_by_pk(car))})
else:
return jsonify(
{'error':
f'{car} does not exist locally, car is case sensitive'}), 400
def need_ikoa_credit(car: str):
try:
db = JavManagerDB()
need = db.get_by_pk(car.upper()).get('need_ikoa_credit', '0') == "1"
print(f'need ikoa credit: {need}')
return need
except DoesNotExist as e:
# for any other error we return False
return False
except Exception as e:
# for any other error we return False
return False
def update_car_ikoa_stat():
car = request.args.get('car').upper()
stat = request.args.get('stat')
need_ikoa_credit = request.args.get('need_ikoa_credit') or "0"
db_conn = JavManagerDB()
db_conn.upcreate_jav({
'car': car,
'stat': stat,
'need_ikoa_credit': need_ikoa_credit
})
return jsonify({'success': 'ok'})
def __init__(self, root_path):
if not os.path.exists(root_path):
raise Exception(f'{root_path} does not exist')
if not os.path.isdir(root_path):
raise Exception(f'{root_path} is not a valid directory for scan')
self.root_path = root_path
self.file_list = []
self.folder_structure = return_default_config_string('folder_structure')
# settings from ini file
self.handle_multi_cds = ( return_default_config_string('handle_multi_cds')== '是' )
self.preserve_subtitle_filename = ( return_default_config_string('preserve_subtitle_filename')== '是' )
self.subtitle_filename_postfix = return_default_config_string('subtitle_filename_postfix').split(',')
self.jav_manage = JavManagerDB()
def local_set_page(page_template: str,
page_num=1,
url_parameter=None,
config=None):
"""
local return func
currently only support stat search
"""
stat_type = parse_qs(page_template)
_stat = str(stat_type.get('stat', [0])[0])
db = JavManagerDB()
print(f'searching jav with stat {_stat}')
# search on both stat string and int
s_result, max_page = db.query_on_filter({'stat': int(_stat)},
page=int(page_num))
for jav_obj in s_result:
# get rid of invalid image url from javdb
if 'jdbimgs' in jav_obj.get('image', ''):
jav_obj.pop('image')
elif 'jdbimgs' in jav_obj.get('img', ''):
jav_obj.pop('img')
if not jav_obj.get('image') and not jav_obj.get('img'):
# need to refresh db to get image
jav_obj.update(find_images(jav_obj['car']))
return s_result, max_page
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify({'error':
f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num)
db_conn = JavManagerDB()
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
return jsonify({'success': {'jav_objs': jav_objs, 'max_page': max_page}})
def find_images():
car = request.args.get('car')
sources = request.args.get('sources')
if not car:
return jsonify({'error': 'cannot find car from request'}), 400
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except (DoesNotExist, TypeError) as e:
# typeerror to catch dict(None)
jav_obj = {'car': car}
# verify sources
if not sources:
sources = return_default_config_string('jav_obj_priority').split(',')
else:
sources = str(sources).split(',')
res = parse_single_jav({'car': car}, sources)
if res != jav_obj:
jav_obj.update(res)
db_conn.upcreate_jav(jav_obj)
return jsonify({'success': jav_obj})
def update_db_jav():
req_data = json.loads(request.get_data() or '{}')
jav_pk = req_data.get('pk')
update_data = req_data.get('data')
if not jav_pk:
return jsonify({'error': 'no pk found in posted json'}), 400
db_conn = JavManagerDB()
try:
current_jav_obj = dict(db_conn.get_by_pk(jav_pk))
except DoesNotExist:
current_jav_obj = {'car': jav_pk}
current_jav_obj.update(update_data)
db_conn.upcreate_jav(current_jav_obj)
return jsonify({'success': dict(current_jav_obj)})
def clean_up_directory():
original_root = request.args.get('original_root') or None
if not original_root:
return jsonify({'error': 'original_root is required parameter'})
db_conn = JavManagerDB()
jav_objs = db_conn.bulk_list()
for jav_obj in jav_objs:
if jav_obj.get('directory') and original_root in jav_obj['directory']:
_temp = jav_obj['directory']
_temp = _temp.replace(original_root, '')
print('updating {} to {}'.format(jav_obj['directory'], _temp))
jav_obj['directory'] = _temp
db_conn.upcreate_jav(dict(jav_obj))
#break
return jsonify({'success': 'ok'})
def get_set_javs():
lib_type = request.args.get('lib_type')
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# check lib type exists
if lib_type not in LIB_MAP:
return jsonify({'error':
f'{lib_type} is not a supported jav library'}), 400
else:
_set_map = LIB_MAP[lib_type]['supported_set']
_search_func = LIB_MAP[lib_type].get('search_func')
_set_func = LIB_MAP[lib_type]['set_func']
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if not search_string:
# parse set without search string
# verify set type
if set_type not in _set_map:
set_type = list(_set_map)[0]
jav_objs, max_page = _set_func(_set_map[set_type], page_num)
else:
# search by supplied string
jav_objs, max_page = _search_func(set_type, search_string, page_num)
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
# we cannot use img stored in db if it was from javdb due to
# dynamically verified id
db_obj = dict(db_conn.get_by_pk(str(jav_obj.get('car'))))
if db_obj.get('img') and 'jdbimgs' in db_obj['img']:
db_obj.pop('img')
jav_obj.update(db_obj)
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify(
{'success': {
'jav_objs': rt_jav_objs,
'max_page': max_page
}})
def rewrite_images():
req_data = json.loads(request.get_data() or '{}')
update_dict = req_data['update_dict']
JavManagerDB().upcreate_jav(update_dict)
file_writer = EmbyFileStructure(return_default_config_string('file_path'))
# we can directly call this since it only writes top level key fields
file_writer.write_images(update_dict, fail_on_error=True)
return jsonify({'success': 'good'})
def search_by_car(car: str, **kwargs):
car = car.upper()
jav_obj = JavLibraryScraper({'car': car}).scrape_jav()
db_conn = JavManagerDB()
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
# use the full image (image key) instead of img (much smaller)
jav_obj['img'] = jav_obj.get('image', '')
return [jav_obj], 1
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if not search_string:
# parse set without search string
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify({'error': f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javdb_set_page(SET_TYPE_MAP[set_type], page_num)
else:
# search by supplied string
search_map = {
'番号': {'function': search_by_car, 'params': {'car': search_string}},
'女优': {'function': search_for_actress, 'params': {
'javlib_actress_code': search_string, 'page_num': page_num}},
'分类': {'function': javdb_set_page, 'params':
{'page_template': 'tags?c6={url_parameter}&page={page_num}',
'page_num': page_num, 'url_parameter': search_string}
},
'系列': {'function': javdb_set_page, 'params':
{'page_template': 'series/{url_parameter}?page={page_num}',
'page_num': page_num, 'url_parameter': search_string}
}
}
# verify set type
if set_type not in search_map:
return jsonify({'error': BackendTranslation()['no_support_set_search'].format(set_type)}), 400
jav_objs, max_page = search_map[set_type]['function'](**search_map[set_type]['params'])
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(
dict(
db_conn.get_by_pk(str(jav_obj.get('car')))
)
)
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify({'success': {'jav_objs': rt_jav_objs, 'max_page': max_page}})
def find_images(car: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except (DoesNotExist, TypeError) as e:
# typeerror to catch dict(None)
jav_obj = {'car': car}
sources = return_default_config_string('jav_obj_priority').split(',')
res = parse_single_jav({'car': car}, sources)
if res != jav_obj:
jav_obj.update(res)
db_conn.upcreate_jav(jav_obj)
return jav_obj
def search_for_actress(javlib_actress_code: str, page_num=1):
search_url = 'actors/{url_parameter}?page={page_num}'
db_conn = JavManagerDB()
# get actress first page
jav_objs, max_page = javdb_set_page(search_url,
page_num=page_num,
url_parameter=javlib_actress_code)
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(
jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jav_objs, max_page
def search_for_actress(javlib_actress_code: str, page_num=1):
"""
This only support javlibrary actress code
"""
search_url = 'vl_star.php?&mode=&s={url_parameter}&page={page_num}'
db_conn = JavManagerDB()
# get actress first page
jav_objs, max_page = javlib_set_page(search_url,
page_num=page_num,
url_parameter=javlib_actress_code)
for jav_obj in jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(
jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jav_objs, max_page
def handle_jav_download(self, car: str, magnet: str):
db_conn = JavManagerDB()
try:
jav_obj = dict(db_conn.get_by_pk(car))
except (DoesNotExist, TypeError) as e:
# typeerror to catch dict(None)
jav_obj = {'car': car}
retry_num = 0
e = None
# send download info to aria2
try:
res = requests.post(self.deluge_address,
json={
'id': 0,
'method': 'core.add_torrent_magnet',
'params': [magnet, {}]
},
cookies=self.cookies).json()
if res.get('result'):
# if everything went well, update stat
jav_obj['stat'] = 4
db_conn.upcreate_jav(jav_obj)
return jav_obj
else:
raise Exception(res.get('error', {}).get('message'))
except Exception as _e:
print_exc()
e = _e
return {
'error':
self.translate_map['oof_general_failure'].format(
car=car, retry_num=retry_num, e=e)
}
def db_stat_lookup(page_num):
# handle db stat 0 look up
db_conn = JavManagerDB()
jav_objs, max_page = db_conn.query_on_filter({'stat': 0},
page=int(page_num))
# need additional info
for jav_obj in jav_objs:
if jav_obj['stat'] != 0:
print('0 stat detected, reindex and redo jav obj lookup')
db_conn.rebuild_index()
return db_stat_lookup(page_num)
if not jav_obj.get('title', None):
_full_info = JavLibraryScraper({
'car': jav_obj['car']
}).scrape_jav()
jav_obj.update(_full_info)
db_conn.upcreate_jav(jav_obj)
jav_obj.setdefault('img', jav_obj.get('image',
'')) # force img key to exist
return jav_objs, max_page
def get_set_javs():
set_type = request.args.get('set_type')
page_num = request.args.get('page_num', 1)
# optional search string
search_string = request.args.get('search_string', '')
_dedupe_car_list = []
rt_jav_objs = []
if set_type == 'personal_wanted':
db_conn = JavManagerDB()
jav_objs, max_page = db_conn.query_on_filter({'stat': 0},
page=int(page_num))
# need additional info
for jav_obj in jav_objs:
if jav_obj['stat'] != 0:
db_conn.rebuild_index()
raise Exception(
'index is not up-to-date and it has been rebuild')
if not jav_obj.get('title', None):
_full_info = JavLibraryScraper({
'car': jav_obj['car']
}).scrape_jav()
jav_obj.update(_full_info)
db_conn.upcreate_jav(jav_obj)
jav_obj.setdefault('img',
jav_obj.get('image',
'')) # force img key to exist
# don't need extra db operations
return jsonify(
{'success': {
'jav_objs': jav_objs,
'max_page': max_page
}})
elif not search_string:
# parse set without search string
# verify set type
if set_type not in SET_TYPE_MAP:
return jsonify(
{'error': f'{set_type} is not a supported set type'}), 400
jav_objs, max_page = javlib_set_page(SET_TYPE_MAP[set_type], page_num)
else:
# search by supplied string
search_map = {
'番号': {
'function': search_by_car,
'params': {
'car': search_string
}
},
'女优': {
'function': search_for_actress,
'params': {
'javlib_actress_code': search_string,
'page_num': page_num
}
},
'分类': {
'function': javlib_set_page,
'params': {
'page_template':
'vl_genre.php?&mode=&g={url_parameter}&page={page_num}',
'page_num': page_num,
'url_parameter': search_string
}
},
}
# verify set type
if set_type not in search_map:
return jsonify(
{'error': f'{set_type} is not a supported search type'}), 400
jav_objs, max_page = search_map[set_type]['function'](
**search_map[set_type]['params'])
for jav_obj in jav_objs:
if jav_obj['car'] not in _dedupe_car_list:
_dedupe_car_list.append(jav_obj['car'])
rt_jav_objs.append(jav_obj)
# looooop through DB
db_conn = JavManagerDB()
for jav_obj in rt_jav_objs:
if db_conn.pk_exist(str(jav_obj.get('car'))):
jav_obj.update(dict(db_conn.get_by_pk(str(jav_obj.get('car')))))
else:
jav_obj['stat'] = 2
db_conn.upcreate_jav(jav_obj)
return jsonify(
{'success': {
'jav_objs': rt_jav_objs,
'max_page': max_page
}})
class EmbyFileStructure:
def __init__(self, root_path):
if not os.path.exists(root_path):
raise Exception(f'{root_path} does not exist')
if not os.path.isdir(root_path):
raise Exception(f'{root_path} is not a valid directory for scan')
self.root_path = root_path
self.file_list = []
self.folder_structure = return_default_config_string(
'folder_structure')
# settings from ini file
self.handle_multi_cds = (
return_default_config_string('handle_multi_cds') == '是')
self.preserve_subtitle_filename = (
return_default_config_string('preserve_subtitle_filename') == '是')
self.subtitle_filename_postfix = return_default_config_string(
'subtitle_filename_postfix').split(',')
self.jav_manage = JavManagerDB()
def write_images(self, jav_obj):
poster_name = POSTER_NAME
fanart_name = FANART_NAME
if 'image' not in jav_obj:
raise Exception('no image field in jav_obj')
image_url = jav_obj['image']
if 'directory' not in jav_obj:
raise Exception('no directory field in jav_obj')
# windows and linux compatible
directory = jav_obj['directory'].replace('/',
os.sep).replace('\\', os.sep)
directory = os.path.join(self.root_path, directory)
# 下载海报的地址 cover
url_obj = urlparse(image_url, scheme='http')
image_ext = os.path.splitext(url_obj.path)[1]
poster_path = os.path.join(directory, poster_name + image_ext)
fanart_path = os.path.join(directory, fanart_name + image_ext)
r = requests.get(url_obj.geturl(), stream=True)
if r.status_code != 200:
raise Exception('Image download failed for {}'.format(
url_obj.geturl()))
with open(fanart_path, 'wb') as pic:
for chunk in r:
pic.write(chunk)
# 裁剪生成 poster
img = Image.open(fanart_path)
w, h = img.size # fanart的宽 高
ex = int(w * 0.52625) # 0.52625是根据emby的poster宽高比较出来的
poster = img.crop((ex, 0, w, h)) # （ex，0）是左下角（x，y）坐标 （w, h)是右上角（x，y）坐标
try:
# quality=95 是无损crop，如果不设置，默认75
poster.save(poster_path, quality=95)
except OSError:
# handle RGBA error
if poster.mode in ('RGBA', 'LA'):
background = Image.new(poster.mode[:-1], poster.size,
'#ffffff')
background.paste(poster, poster.split()[-1])
poster = background
poster.save(poster_path, quality=95)
return
def write_nfo(self, jav_obj: dict, verify=False):
if 'file_name' not in jav_obj:
raise Exception('no file_name field in jav_obj')
file_name = jav_obj['file_name']
if 'directory' not in jav_obj:
raise Exception('no directory field in jav_obj')
# windows and linux compatible
directory = jav_obj['directory'].replace('/',
os.sep).replace('\\', os.sep)
directory = os.path.join(self.root_path, directory)
if verify:
# verify there is a corresponding video file
directory_files = os.listdir(directory)
correct_file_exists = False
for _filename in directory_files:
if _filename == os.path.splitext(file_name)[0] + '.nfo':
continue
if _filename.startswith(os.path.splitext(file_name)[0]):
correct_file_exists = True
break
if not correct_file_exists:
raise Exception(
'there is no correct file exists in {} for {}'.format(
os.path.splitext(file_name)[0], directory))
nfo_file_name = os.path.splitext(file_name)[0] + '.nfo'
nfo_path = os.path.join(directory, nfo_file_name)
with codecs.open(nfo_path, 'w', 'utf-8') as f:
if not jav_obj.get('title'):
#import ipdb; ipdb.set_trace()
raise Exception(
'[FATAL ERROR] There is no valid title for {}'.format(
jav_obj['car']))
f.write(
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\" ?>\n"
"<movie>\n"
" <plot>{plot}</plot>\n"
" <title>{title}</title>\n"
" <director>{director}</director>\n"
" <rating>{score}</rating>\n"
# " <criticrating>" + criticrating + "</criticrating>\n"
" <year>{year}</year>\n"
" <release>{release_date}</release>\n"
" <runtime>{length}</runtime>\n"
" <studio>{studio}</studio>\n"
" <id>{car}</id>\n".format(
plot=jav_obj.get('plot', ''),
title=jav_obj.get('title', ''),
director=jav_obj.get('director', ''),
score=jav_obj.get('score', ''),
year=jav_obj.get('year', ''),
release_date=jav_obj.get('release_date', ''),
length=jav_obj.get('length', ''),
studio=jav_obj.get('studio', ''),
car=jav_obj.get('car', '')))
for i in jav_obj.get('genres', []):
f.write(" <genre>" + i + "</genre>\n")
f.write(" <genre>片商：{}</genre>\n".format(jav_obj.get(
'studio', '')))
for i in jav_obj.get('genres', []):
f.write(" <tag>" + i + "</tag>\n")
f.write(" <tag>片商：{}</tag>\n".format(jav_obj.get('studio', '')))
for i in jav_obj.get('all_actress', []):
f.write(
" <actor>\n <name>" + i +
"</name>\n <type>Actor</type><role>{}</role>\n </actor>\n"
.format(self.return_actor_role()))
f.write("</movie>\n")
@staticmethod
def return_actor_role():
language_map = {'cn': '女优', 'en': 'P**n Star'}
language = return_default_config_string('display_language')
return language_map[language]
def extract_subtitle_postfix_filename(self, file_name: str):
subtitle_postfix = ''
if self.preserve_subtitle_filename:
for postfix in self.subtitle_filename_postfix:
if file_name.endswith(postfix):
print(f'find subtitle postfix {postfix} in {file_name}')
#subtitle_postfix = postfix
subtitle_postfix = '-C'
file_name = file_name[:-len(postfix)]
break
return subtitle_postfix, file_name
def extract_CDs_postfix_filename(self, file_name: str):
"""
Cannot have -C or -CD2 in the sametime
"""
allowed_postfixes = {
r'^(.+?)([ABab])$': {
'a': 'cd1',
'b': 'cd2'
},
r'^(.+?)(CD\d|cd\d)$': None,
}
cd_postfix = ''
if not self.handle_multi_cds:
return cd_postfix, file_name
for pattern, part_map in allowed_postfixes.items():
file_name_match = re.match(pattern, file_name)
if file_name_match:
file_name = file_name_match.group(1)
cd_postfix = file_name_match.group(2).lower()
print(file_name, cd_postfix)
if part_map:
cd_postfix = part_map[cd_postfix]
break
return cd_postfix, file_name
def rename_single_file(self,
file_name: str,
name_pattern=DEFAULT_FILENAME_PATTERN):
subtitle_postfix, file_name = self.extract_subtitle_postfix_filename(
file_name)
cd_postfix, file_name = self.extract_CDs_postfix_filename(file_name)
name_group = re.search(name_pattern, file_name)
name_digits = name_group.group('digit')
# only keep 0 under 3 digits
# keep 045, 0830 > 830, 1130, 0002 > 002, 005
if name_digits.isdigit():
name_digits = str(int(name_digits))
while len(name_digits) < 3:
name_digits = '0' + name_digits
new_file_name = name_group.group(
'pre') + '-' + name_digits + subtitle_postfix + cd_postfix
return new_file_name
def rename_directory_preview(self, name_pattern=None):
# apply default name pattern
if not name_pattern:
name_pattern = DEFAULT_FILENAME_PATTERN
res = []
for ind_file in os.listdir(self.root_path):
if str(ind_file) == '.DS_Store' or str(ind_file).startswith('.'):
continue
ind_file_name, ind_ext = os.path.splitext(ind_file)
print(ind_file)
try:
if ind_file_name.startswith('hjd2048.com'):
ind_file_name = ind_file_name[11:]
elif ind_file_name.startswith('[Thz.la]'):
ind_file_name = ind_file_name[8:]
elif '_' in ind_file_name:
ind_file_name = ind_file_name.replace('_', '-')
if ind_file_name.startswith('T28'):
# TODO: might need to rename this
continue
elif ind_file_name.startswith('T-28'):
ind_file_name = ind_file_name.replace('T-28', 'T28-')
t28_pattern = r'^.*?(?P<pre>T28)\W*(?P<digit>\d{1,5}).*?$'
new_file_name = self.rename_single_file(
ind_file_name, t28_pattern) + ind_ext
else:
# normal case
new_file_name = self.rename_single_file(
ind_file_name, name_pattern) + ind_ext
except Exception as e:
traceback.print_exc()
res.append(
{'file_name': f'cannot process {ind_file} due to {e}'})
continue
# log before rename
if ind_file == new_file_name:
res.append({'file_name': ind_file})
else:
res.append({
'file_name': ind_file,
'new_file_name': new_file_name
})
return res
@staticmethod
def rename_directory(path, file_objs):
for each_file in file_objs:
if not each_file.get('new_file_name'):
# not rename unnecessary files
continue
try:
ind_file = each_file['file_name']
new_file_name = each_file['new_file_name']
# rename
os.rename(os.path.join(path, ind_file),
os.path.join(path, new_file_name))
yield f'renamed {ind_file} to {new_file_name}'
except Exception as e:
yield f'failed to renamed {ind_file} to due to {e}'
def scan_new_root_path(self):
"""
This function is used to scan unprocessed video files
which means it only append file object from root path and
will not return any sub-directory files.
"""
# fill file list from scan
for file_name in os.listdir(self.root_path):
# don't care about directory
# also ignore file starts with . (auto generated by macos)
if os.path.isdir(os.path.join(self.root_path, file_name)) or \
os.path.splitext(file_name)[0].startswith('.'):
continue
# ini jav obj
# fill car
postfix, car_str = self.extract_subtitle_postfix_filename(
os.path.splitext(file_name)[0])
_, car_str = self.extract_CDs_postfix_filename(car_str)
jav_obj = {'file_name': file_name, 'car': car_str}
if postfix:
jav_obj.setdefault('genres', []).append('中字')
self.file_list.append(jav_obj)
def scan_emby_root_path(self):
"""
This function is used to scan processed emby profile, so it will only
scan correct directories and their nfo files
"""
for (root, dirs, files) in os.walk(self.root_path):
for each_file in files:
# we only process nfo file from here
# also ignore file starts with . (auto generated by macos)
if os.path.splitext(each_file)[1] == '.nfo' and \
not os.path.splitext(each_file)[0].startswith('.'):
nfo_obj = EmbyNfo()
nfo_obj.parse_emby_nfo(os.path.join(root, each_file))
# we only save Relative path to the root in db to make the db work across different machine
nfo_obj.jav_obj['directory'] = os.path.relpath(
root, self.root_path)
nfo_obj.jav_obj['stat'] = 3
self.jav_manage.upcreate_jav(nfo_obj.jav_obj)
self.file_list.append(nfo_obj.jav_obj)
print('scanned {}'.format(nfo_obj.jav_obj['directory']))
def create_new_folder(self, jav_obj: dict):
# file_name has to be in incoming jav_obj
if 'file_name' not in jav_obj:
raise Exception(f'file_name has to be in incoming jav object')
file_name = jav_obj['file_name']
if not os.path.exists(os.path.join(self.root_path, file_name)):
raise Exception(f'{file_name} does not exist')
# configure all necessary folders
try:
new_full_path = os.path.join(
self.root_path, self.folder_structure.format(**jav_obj))
except KeyError:
raise KeyError(
'required fields not filled for path {} and parsed {}'.format(
self.folder_structure, jav_obj.keys()))
os.makedirs(new_full_path, exist_ok=True)
print(f'{new_full_path} created')
# we only save relative directory excluding root
# since root is configurable among different machines
jav_obj['directory'] = self.folder_structure.format(**jav_obj)
return jav_obj
def create_folder_for_existing_jav(self, jav_obj: dict):
# file_name has to be in incoming jav_obj
if 'file_name' not in jav_obj:
raise Exception(f'file_name has to be in incoming jav object')
file_name = jav_obj['file_name']
if not os.path.exists(
os.path.join(self.root_path, jav_obj['directory'], file_name)):
raise Exception('{} does not exist'.format(
os.path.join(self.root_path, jav_obj['directory'], file_name)))
# configure all necessary folders
try:
new_full_path = os.path.join(
self.root_path, self.folder_structure.format(**jav_obj))
except KeyError:
raise KeyError(
'required fields not filled for path {} and parsed {}'.format(
self.folder_structure, jav_obj.keys()))
os.makedirs(new_full_path, exist_ok=True)
print(f'{new_full_path} created')
# we only save relative directory excluding root
# since root is configurable among different machines
jav_obj['old_directory'] = deepcopy(jav_obj['directory'])
jav_obj['directory'] = self.folder_structure.format(**jav_obj)
print('{} >> {}'.format(jav_obj['old_directory'],
jav_obj['directory']))
return jav_obj
@staticmethod
def find_corresponding_video_file(file_name: str, root_path: str,
relative_directory: str):
"""
This function will attempt to find nfo file's corresponding video file
"""
if not file_name.endswith('.nfo'):
return file_name
filename, _ = os.path.splitext(file_name)
for f in os.scandir(os.path.join(root_path, relative_directory)):
_f, _ext = os.path.splitext(f.name)
if _f == filename and _ext != '.nfo':
return f.name
# by default just return input file name since nothing is found
print('[WARNING] cannot find video file for {} in {}'.format(
file_name, os.path.join(root_path, relative_directory)))
return file_name
def move_existing_file(self, jav_obj: dict):
# file_name has to be in incoming jav_obj
if 'file_name' not in jav_obj or 'directory' not in jav_obj:
raise Exception(
f'required file_name or directoy has to be in incoming {jav_obj} object'
)
if 'old_directory' not in jav_obj:
raise Exception(
f'required old_directoy has to be in incoming {jav_obj} object'
)
if jav_obj['directory'].replace('/', os.sep).replace('\\', os.sep) == \
jav_obj['old_directory'].replace('/', os.sep).replace('\\', os.sep):
print(
'old & new directories {} are already the same, no need to move'
.format(jav_obj['directory']))
return jav_obj
file_name = jav_obj['file_name']
# join relative directory with root
# windows and linux compatible
directory = jav_obj['directory'].replace('/',
os.sep).replace('\\', os.sep)
new_full_path = os.path.join(self.root_path, directory)
old_directory = jav_obj['old_directory'].replace('/', os.sep).replace(
'\\', os.sep)
# need to find corresponding video files
_move_file_name = self.find_corresponding_video_file(
file_name, self.root_path, old_directory)
if not os.path.exists(
os.path.join(self.root_path, old_directory, _move_file_name)):
raise Exception(f'{_move_file_name} does not exist')
try:
os.rename(
os.path.join(self.root_path, old_directory, _move_file_name),
os.path.join(new_full_path, _move_file_name))
except FileExistsError:
print('[WARNING] {} already exists in target path'.format(
os.path.join(new_full_path, _move_file_name)))
# default to exists locally
jav_obj.setdefault('stat', 3)
# write to db
self.jav_manage.upcreate_jav(jav_obj)
print('move {} to {}'.format(
os.path.join(self.root_path, old_directory, _move_file_name),
os.path.join(new_full_path, _move_file_name)))
return jav_obj
def put_processed_file(self, jav_obj: dict):
# file_name has to be in incoming jav_obj
if 'file_name' not in jav_obj or 'directory' not in jav_obj:
raise Exception(
f'required file_name or directoy has to be in incoming {jav_obj} object'
)
file_name = jav_obj['file_name']
# join relative directory with root
# windows and linux compatible
directory = jav_obj['directory'].replace('/',
os.sep).replace('\\', os.sep)
new_full_path = os.path.join(self.root_path, directory)
if not os.path.exists(os.path.join(self.root_path, file_name)):
raise Exception(f'{file_name} does not exist')
os.rename(os.path.join(self.root_path, file_name),
os.path.join(new_full_path, file_name))
# default to exists locally
jav_obj.setdefault('stat', 3)
# write to db
self.jav_manage.upcreate_jav(jav_obj)
print('move {} to {}'.format(os.path.join(self.root_path, file_name),
os.path.join(new_full_path, file_name)))
return jav_obj
def jav_manage(self):
# need to dynamically return this so this is not called everytime init the class
# may encounter weird error
return JavManagerDB()
def rebuild_db_index():
JavManagerDB().rebuild_index()
return jsonify({'success': 'index rebuilt for stat'})
