def json_edit_func(): from mylib.ex.fstk import write_json_file from mylib.ex.fstk import read_json_file args = rtd.args file = args.file or list_files(clipboard)[0] indent = args.indent delete = args.delete item_l = args.item d = read_json_file(file) if delete: def handle(key, value): if key in d: if value: if d[key] == value: del d[key] else: del d[key] else: def handle(key, value): d[key] = value for item in item_l: k, v = map(eval_or_str, item.split('=', maxsplit=1)) handle(k, v) write_json_file(file, d, indent=indent)
def update_json_file(): from mylib.ex.fstk import write_json_file from mylib.ex.fstk import read_json_file args = rtd.args old, new = args.old, args.new d = read_json_file(old) d.update(read_json_file(new)) write_json_file(old, d, indent=args.indent)
def download_pixiv_fanbox_creator(creator_id, root_dir='.', fanbox_api: PixivFanboxAPI = None, download_pool: DownloadPool = None, retry=-1, **kwargs_for_requests): download_params = {'retry': retry, **kwargs_for_requests} fanbox_api = fanbox_api or PixivFanboxAPI(**kwargs_for_requests) download_pool = download_pool or DownloadPool() creator = fanbox_api.get_creator_info(creator_id) creator_id = creator['creatorId'] prefix = '{}.'.format(creator_id) creator['plans'] = fanbox_api.list_sponsor_plan_of_creator(creator_id) profile_images = [ i for i in creator['profileItems'] if i['type'] == 'image' ] creator_folder = sanitize_xu(pixiv_fanbox_creator_folder(creator)) os.makedirs(os.path.join(root_dir, creator_folder), exist_ok=True) write_json_file(os.path.join(root_dir, creator_folder, creator_id + '.json'), creator, indent=4) url = creator['user']['iconUrl'] file = prefix + 'icon.' + os.path.split(url)[-1] filepath = os.path.join(root_dir, creator_folder, file) download_pool.put_download_in_queue(url, filepath, **download_params) url = creator['coverImageUrl'] file = prefix + 'cover.' + os.path.split(url)[-1] filepath = os.path.join(root_dir, creator_folder, file) download_pool.put_download_in_queue(url, filepath, **download_params) n_width = width_of_int(len(profile_images)) n = 0 for i in profile_images: n += 1 url = i['imageUrl'] file = prefix + 'profile{}.{}'.format( str(n).zfill(n_width), os.path.split(url)[-1]) filepath = os.path.join(root_dir, creator_folder, file) download_pool.put_download_in_queue(url, filepath, **download_params) for i in creator['plans']: url = i['coverImageUrl'] file = prefix + 'plan{}.{}.{}'.format(i['fee'], i['title'], os.path.split(url)[-1]) filepath = os.path.join(root_dir, creator_folder, file) download_pool.put_download_in_queue(url, filepath, **download_params) download_pool.start_queue_loop() download_pool.put_end_of_queue()
def download_pixiv_fanbox_post(post_or_id: PixivFanboxPost or dict or str or int, root_dir='.', fanbox_api: PixivFanboxAPI = None, download_pool: DownloadPool = None, retry=-1, **kwargs_for_requests): download_pool = download_pool or DownloadPool() if isinstance(post_or_id, PixivFanboxPost): post = post_or_id elif isinstance(post_or_id, dict): post = PixivFanboxPost(post_or_id) elif isinstance(post_or_id, (str, int)): fanbox_api = fanbox_api or PixivFanboxAPI(**kwargs_for_requests) post = fanbox_api.get_post_info(post_or_id) else: raise TypeError( "`post_or_id` must be PixivFanboxPost, dict, str, or int, not {}". format(type(post_or_id))) creator_folder = sanitize_xu(pixiv_fanbox_creator_folder(post.__data__)) creator_id = post.creator_id prefix = '{}.'.format(creator_id) post_folder = sanitize_xu( '[{user[name]} ({creatorId})] {title} (pixiv fanbox {id})'.format( **post.__data__)) os.makedirs(os.path.join(root_dir, creator_folder, post_folder), exist_ok=True) file = prefix + '{}.json'.format(post.id) filepath = os.path.join(root_dir, creator_folder, post_folder, file) write_json_file(filepath, post.__data__, indent=4) url = post.cover_image_url if url: file = prefix + '{}.cover.{}'.format(post.id, os.path.split(url)[-1]) filepath = os.path.join(root_dir, creator_folder, post_folder, file) download_pool.put_download_in_queue(url, filepath, retry, **kwargs_for_requests) n_width = width_of_int(len(post.images)) n = 0 for image in post.images: n += 1 file = prefix + '{}.{}.{}.{}'.format(post.id, str(n).zfill(n_width), image.id, image.extension) filepath = os.path.join(root_dir, creator_folder, post_folder, file) download_pool.put_download_in_queue(image.original_url, filepath, retry, **kwargs_for_requests) download_pool.start_queue_loop() download_pool.put_end_of_queue()
def list_several_cloudflare_ipaddr(file, hostname, as_list, isp): from mylib.sites.misc import get_cloudflare_ipaddr_hostmonit from mylib.ex.fstk import write_json_file from pprint import pformat data = get_cloudflare_ipaddr_hostmonit() info: dict = data['info'] if isp: info = {isp: info[isp]} if as_list: lines = [] for ip_isp, ip_l in info.items(): lines.append(f'# {ip_isp}') for ip_d in ip_l: li = ip_d['ip'] if hostname: li = f'{li} {hostname}' lines.append(li) output = '\r\n'.join(lines) else: output = pformat(info) if file: write_json_file(file, data, indent=4) clipboard.set(output) print(output)
def move_into_dir(src, dst, pattern, alias, dry_run, sub_dir): from mylib.ex.ostk import fs_move_cli from mylib.easy.text import find_words from mylib.ex.tui import prompt_choose_number, prompt_confirm conf_file = fstk.make_path('~', '.config', 'fs.put_in_dir.json', user_home=True) conf = fstk.read_json_file(conf_file) or {'dst_map': {}} dst_map = conf['dst_map'] if pattern: def filename_words(fn: str): return find_words(' '.join(re.findall(pattern, fn))) else: filename_words = find_words if alias is None: pass elif not alias: for k, v in dst_map.items(): print(f'{k}={v}') else: for a in alias: try: k, v = a.split('=', maxsplit=1) except ValueError: k, v = None, None if v: dst_map[k] = v print(f'{k}={v}') elif k and k in dst_map: del dst_map[k] print(f'{k}=') else: print(f'{a}={dst_map.get(a, "")}') fstk.write_json_file(conf_file, conf, indent=4) if not dst: return dst = dst_map.get(dst, dst) sub_dirs_l = next(os.walk(dst))[1] __ = [] for sub_dir_basename in sub_dirs_l: if re.fullmatch(r'#\w+=', sub_dir_basename): for sub_sub_dir_basename in next( os.walk(path_join(dst, sub_dir_basename)))[1]: __.append(path_join(sub_dir_basename, sub_sub_dir_basename)) else: __.append(sub_dir_basename) sub_dirs_l = __ if os.path.isfile(dst): print(f'! {dst} is file (should be directory)', file=sys.stderr) exit(1) os.makedirs(dst, exist_ok=True) db_path = fstk.make_path(dst, '__folder_name_words__.db') db = mylib.ex.fstk.read_sqlite_dict_file(db_path) db = {k: v for k, v in db.items() if k in sub_dirs_l} sub_dirs_d = { sd_bn: set(find_words(sd_bn.lower())) for sd_bn in sub_dirs_l if sd_bn not in db } # sd_bn: sub-dir basename sub_dirs_d.update(db) sub_dirs_d = {k: sub_dirs_d[k] for k in sorted(sub_dirs_d)} for ss in src: for s in fstk.path_or_glob(ss): tui_lp.d() print(s) tui_lp.l() if sub_dir: similar_d = { basename: words_set & set(filename_words(os.path.basename(s).lower())) for basename, words_set in sub_dirs_d.items() } similar_d = {k: v for k, v in similar_d.items() if v} similar_l = sorted(similar_d, key=lambda x: similar_d[x], reverse=True) if similar_l: target_dir_name = prompt_choose_number( f'Select probable folder:', similar_l) tui_lp.l() else: target_dir_name = None if not target_dir_name: keywords = input( 'Input custom keywords or leave it empty: ') if keywords: similar_d = { basename: words_set & set(filename_words(keywords.lower())) for basename, words_set in sub_dirs_d.items() } similar_d = {k: v for k, v in similar_d.items() if v} similar_l = sorted(similar_d, key=lambda x: similar_d[x], reverse=True) if similar_l: target_dir_name = prompt_choose_number( f'Select probable folder for\n{keywords}:', similar_l) tui_lp.l() target_dir_name = target_dir_name or input(f'Create folder: ') if target_dir_name: sub_dirs_d[target_dir_name] = set( find_words(target_dir_name.lower())) dir_path = fstk.make_path(dst, target_dir_name) if not dry_run: os.makedirs(dir_path, exist_ok=True) else: dir_path = dst else: dir_path = dst d = fstk.make_path(dir_path, os.path.basename(s)) if os.path.exists(d): if not prompt_confirm(f'Overwrite {d}?', default=False): continue if not dry_run: fs_move_cli(s, d) print(f'{s} -> {d}') mylib.ex.fstk.write_sqlite_dict_file(db_path, sub_dirs_d, update_only=True)
def __set_conf__(self, **kwargs): conf = self.__get_conf__() conf.update(kwargs) write_json_file(self._conf_file, conf, indent=4)
def rename_dialog(src: str): import PySimpleGUIQt as G ske = PySimpleGUISpecialKeyEvent() conf_file = real_join_path('~', '.config/rename_dialog.json') root = 'root' fname = 'fname' ext = 'ext' key_new_root = 'key_new_root' key_new_base = 'key_new_base' ok = 'OK' cancel = 'Cancel' pattern = 'pattern' replace = 'replace' substitute = 'substitute' save_replace = 'save_replace' save_pattern = 'save_pattern' add_root = 'add_root' rename_info_file = 'rename_info_file' bytes_count = 'bytes_count' title = 'Rename - {}'.format(src) h = None conf = read_json_file(conf_file, default={pattern: [''], replace: ['']}) tmp_pl = conf[pattern] or [''] tmp_rl = conf[replace] or [''] old_root, old_base = os.path.split(src) old_fn, old_ext = os.path.splitext(old_base) info_file_base = [ f for f in os.listdir(old_root) if f.endswith('.info') and ( f.startswith(old_fn) or old_fn.startswith(f.rstrip('.info'))) ] has_info = True if info_file_base else False @deco_factory_retry(Exception, 0, enable_default=True, default=None) def re_sub(): return re.sub(data[pattern], data[replace], data[fname] + data[ext]) def count_name_bytes(name: str): d = {} try: c, b = encode_default_locale(name) d[c] = len(b) except UnicodeEncodeError: pass u8 = 'utf-8' if u8 not in d: try: c, b = encode_default_locale(name, u8) d[c] = len(b) except UnicodeEncodeError: pass return f'Basename Length: {len(name)}, {", ".join([f"{k.upper()} {v} bytes" for k, v in d.items()])}' # sg.theme('SystemDefaultForReal') layout = [[G.T(src, key='src')], [G.HorizontalSeparator()], [ G.I(old_fn, key=fname, focus=True), G.I(old_ext, key=ext, size=(42, h)) ], [ G.I(old_root, key=root), G.B('+', key=add_root, size=(20, h)), G.FolderBrowse('...', target=root, initial_folder=old_root, size=(20, h)) ], [G.HorizontalSeparator()], [G.T('Regular Expression Pattern & Replacement')], [ G.T(size=(0, h)), G.Drop(tmp_pl, key=pattern, enable_events=True, text_color='blue'), G.CB('', default=True, key=save_pattern, enable_events=True, size=(15, h)), G.Drop(tmp_rl, key=replace, enable_events=True, text_color='blue'), G.CB('', default=True, key=save_replace, enable_events=True, size=(15, h)), G.B('Go', key=substitute, size=(25, h)) ], [G.HorizontalSeparator()], [G.I(old_root, key=key_new_root)], [G.I(old_base, key=key_new_base)], [ G.Submit(ok, size=(10, 1)), G.Stretch(), G.T(count_name_bytes(old_base), key=bytes_count), G.Stretch(), G.Cancel(cancel, size=(10, 1)) ]] if has_info: info_file_base = info_file_base[0] info_filepath = os.path.join(old_root, info_file_base) with open(info_filepath, encoding='utf8') as f: info = f.read() layout.insert(2, [ G.CB(info_file_base, default=True, key=rename_info_file, enable_events=True) ]) layout.insert(2, [G.ML(info, key='ML')]) layout.insert(4, [G.HorizontalSeparator()]) ensure_sigint_signal() window = G.Window(title, layout, return_keyboard_events=True, finalize=True, font='arial 10', element_padding=(1, 1)) window.bring_to_front() ml = window.find_element('ML', silent_on_error=True) if ml: ml.update(readonly=True) loop = True data = { fname: old_fn, ext: old_ext, pattern: tmp_pl[0], replace: tmp_rl[0], root: old_root, key_new_root: '', key_new_base: '' } while loop: dst_from_data = os.path.join(data[key_new_root], data[key_new_base]) try: tmp_fname = re_sub() or data[fname] + data[ext] dst = os.path.realpath(os.path.join(data[root], tmp_fname)) except TypeError: dst = src if dst != dst_from_data: nr, nb = os.path.split(dst) window[key_new_root].update(nr) window[key_new_base].update(nb) window[bytes_count].update(count_name_bytes(nb)) event, data = window.read() for k in (root, fname, ext, key_new_root, key_new_base): window[k].update(text_color=None) cur_p = data[pattern] cur_r = data[replace] if event == ske.esc: loop = False elif event == add_root: os.makedirs(data[root], exist_ok=True) elif event == substitute: data[fname], data[ext] = os.path.splitext( re_sub() or data[fname] + data[ext]) window[fname].update(data[fname]) window[ext].update(data[ext]) elif event == save_pattern: if data[save_pattern]: conf[pattern].insert(0, cur_p) conf[pattern] = dedup_list(conf[pattern]) else: conf[pattern] = remove_from_list(conf[pattern], [cur_p]) elif event == save_replace: if data[save_replace]: conf[replace].insert(0, cur_r) conf[replace] = dedup_list(conf[replace]) else: conf[replace] = remove_from_list(conf[replace], [cur_r]) elif event == pattern: window[save_pattern].update(value=cur_p in conf[pattern]) elif event == replace: window[save_replace].update(value=cur_r in conf[replace]) elif event == ok: try: shutil.move(src, dst) if has_info: if data[rename_info_file]: shutil.move(info_filepath, os.path.splitext(dst)[0] + '.info') loop = False except FileNotFoundError: for k in (root, fname, ext): window[k].update(text_color='red') except FileExistsError: for k in (key_new_root, key_new_base): window[k].update(text_color='red') except OSError as e: G.PopupError(str(e)) elif event in (None, cancel): loop = False else: ... else: write_json_file(conf_file, conf, indent=0) window.close()
def ehviewer_images_catalog(root_dir, *, dry_run: bool = False, db_json_path: str = 'ehdb.json'): logger = logging.get_logger('ehvimg', fmt=logging.LOG_FMT_MESSAGE_ONLY) logmsg_move = '* move {} -> {}' logmsg_skip = '# skip {}' logmsg_data = '+ /g/{}/{}' logmsg_err = '! {}' if os.path.isfile(db_json_path): logger.info('@ using DB file: {}'.format(db_json_path)) db = fstk.read_json_file(db_json_path) db = {int(k): v for k, v in db.items()} else: db = {} with fstk.ctx_pushd(root_dir): not_found_gid_token = [] files = [] for f in next(os.walk('.'))[-1]: try: g = EHentaiGallery(f, logger=logger) except ValueError: logger.info(logmsg_skip.format(f)) continue if g.gid not in db and (g.gid, g.token) not in not_found_gid_token: not_found_gid_token.append((g.gid, g.token)) print(logmsg_data.format(g.gid, g.token)) files.append(f) if not_found_gid_token: print('... RETRIEVE GALLERY DATA FROM E-HENTAI API ...') print('... IT WILL TAKE A LONG TIME ...') eh_api = EHentaiAPI() for d in eh_api.get_gallery_data(not_found_gid_token): db[d['gid']] = d fstk.write_json_file(db_json_path, db, indent=4) for f in files: g = EHentaiGallery(f, logger=logger) d = db[g.gid] creators = [] title = d['title'].strip() try: core_title = find_core_title(title) or '__INVALID_CORE_TITLE__' core_title_l = re.findall(r'[\w]+[\-+\']?[\w]?', core_title) if title[:1] + title[-1:] == '[]': creators.append(title[1:-1].strip()) except AttributeError: print(logmsg_err.format(title)) raise comic_magazine_title = None if core_title_l and core_title_l[0].lower() == 'comic': comic_magazine_title_l = [] for s in core_title_l[1:]: if re.match(r'^\d+', s): break elif re.match(r'^(?:vol|no\.|#)(.*)$', s.lower()): break else: comic_magazine_title_l.append(s) if comic_magazine_title_l: comic_magazine_title = 'COMIC ' + ' '.join(comic_magazine_title_l) tags = d['tags'] if 'artist' in tags: creators = tags['artist'] elif 'group' in tags: creators = tags['group'] else: creators = guess_creators_from_ehentai_title(title) if creators: core_title = title # todo: clean below code block if guess_creators_from_ehentai_title work well # for m in ( # re.match(r'^(?:\([^)]+\))\s*\[([^]]+)]', title), # re.match(r'^\[(?:pixiv|fanbox|tumblr|twitter)]\s*(.+)\s*[(\[]', title, flags=re.I), # re.match(r'^\W*artist\W*(\w.*)', title, flags=re.I), # ): # if m: # m1 = m.group(1).strip() # if m1: # if '|' in m1: # creators = [e.strip() for e in m1.split('|')] # else: # creators = [m1] # core_title = title # break if comic_magazine_title: folder = comic_magazine_title.replace('COMIC X-E ROS', 'COMIC X-EROS') elif creators: if len(creators) > 3: folder = VARIOUS else: folder = ', '.join(creators) else: folder = UNKNOWN # print(f': {title}') # DEBUG # print(f': {core_title}') # DEBUG sub_folder = fstk.make_path(fstk.sanitize_xu200(folder), f'{fstk.sanitize_xu200(core_title)} {g.gid}-{g.token}') parent, basename = os.path.split(f) no_ext, ext = os.path.splitext(basename) no_ext = fstk.sanitize_xu240(no_ext.split()[-1]) new_path = fstk.make_path(sub_folder, no_ext + ext) logger.info(logmsg_move.format(f, new_path)) if not dry_run: os.makedirs(sub_folder, exist_ok=True) shutil.move(f, new_path)