def start(subscription, api_type, api_path, site_name, json_settings): metadata = getattr(subscription.scraped, api_type) download_info = subscription.download_info root_directory = download_info["directory"] date_format = json_settings["date_format"] text_length = json_settings["text_length"] reformats = {} reformats["metadata_directory_format"] = json_settings[ "metadata_directory_format"] reformats["file_directory_format"] = json_settings["file_directory_format"] reformats["filename_format"] = json_settings["filename_format"] username = subscription.username option = {} option["site_name"] = site_name option["api_type"] = api_type option["username"] = username option["date_format"] = date_format option["maximum_length"] = text_length option["directory"] = root_directory formatted = format_types(reformats).check_unique() unique = formatted["unique"] for key, value in reformats.items(): key2 = getattr(unique, key)[0] reformats[key] = value.split(key2, 1)[0] + key2 print print a, base_directory, c = prepare_reformat(option, keep_vars=True).reformat(reformats) download_info["base_directory"] = base_directory print all_files = [] for root, subdirs, files in os.walk(base_directory): x = [os.path.join(root, x) for x in files] all_files.extend(x) for media_type, value in metadata.content: if media_type == "Texts": continue for status, value2 in value: fixed, new_directories = fix_directories(value2, root_directory, site_name, api_path, media_type, username, all_files, json_settings) for new_directory in new_directories: directory = os.path.abspath(new_directory) os.makedirs(directory, exist_ok=True) fixed2 = fix_metadata(fixed) setattr(value, status, fixed2) setattr( metadata.content, media_type, value, ) return metadata
def start(Session, parent_type, api_type, api_path, site_name, subscription, folder, json_settings): api_table = folder.api_table media_table = folder.media_table database_session = Session() result = database_session.query(api_table).all() metadata = getattr(subscription.scraped, api_type) download_info = subscription.download_info root_directory = download_info["directory"] date_format = json_settings["date_format"] text_length = json_settings["text_length"] reformats = {} reformats["metadata_directory_format"] = json_settings[ "metadata_directory_format"] reformats["file_directory_format"] = json_settings["file_directory_format"] reformats["filename_format"] = json_settings["filename_format"] username = subscription.username option = {} option["site_name"] = site_name option["api_type"] = api_type option["username"] = username option["date_format"] = date_format option["maximum_length"] = text_length option["directory"] = root_directory formatted = format_types(reformats).check_unique() unique = formatted["unique"] for key, value in reformats.items(): key2 = getattr(unique, key)[0] reformats[key] = value.split(key2, 1)[0] + key2 print print a, base_directory, c = prepare_reformat(option, keep_vars=True).reformat(reformats) download_info["base_directory"] = base_directory print all_files = [] for root, subdirs, files in os.walk(base_directory): x = [os.path.join(root, x) for x in files] all_files.extend(x) fixed, new_directories = fix_directories(result, all_files, database_session, folder, site_name, parent_type, api_type, username, root_directory, json_settings) database_session.close() return metadata
def fix(config={}): added = [] changed = [] fix = [] settings = {} global_user_agent = "" for key, value in config.items(): if key == "settings": settings = value auto_profile_choice = settings.pop("auto_profile_choice", None) socks5_proxies = settings.pop("socks5_proxy", None) if socks5_proxies: fixed_socks5_proxies = [] for socks5_proxy in socks5_proxies: fixed_socks5_proxy = f"socks5://{socks5_proxy}" fixed_socks5_proxies.append(fixed_socks5_proxy) settings["proxies"] = fixed_socks5_proxies global_user_agent = settings.pop("global_user_agent", None) if key == "supported": for key2, value2 in value.items(): temp_auth = value2.pop("auth", None) if temp_auth: q = os.path.abspath(".settings") backup_config_filepath = os.path.join( q, "config_backup.json") print( f"LEGACY CONFIG FOUND, BACKING IT UP AND CREATING A NEW ONE. ({backup_config_filepath})" ) export_json(backup_config_filepath, config) print temp_auth["user_agent"] = global_user_agent auth = {} temp_auth = auth_details(temp_auth).__dict__ auth["auth"] = temp_auth if "profile_directories" in settings: dpd = settings["profile_directories"][0] default_profile_directory = os.path.join( os.path.abspath(dpd), key2, "default") os.makedirs(default_profile_directory, exist_ok=True) profile_auth_filepath = os.path.join( default_profile_directory, "auth.json") export_json(profile_auth_filepath, auth) print( f"{profile_auth_filepath} HAS BEEN CREATED, CHECK IF IT'S CORRECT." ) print for key3, settings in value2.items(): if key3 == "settings": settings["text_length"] = int(settings["text_length"]) re = settings.pop("download_paths", None) if re: settings["download_directories"] = re string = f"download_paths to download_directories in {key2}" changed.append(string) re = settings.get("metadata_directory_format", None) if not re: settings[ "metadata_directory_format"] = "{site_name}/{username}/Metadata" string = f"metadata_directory_format in {key2}" added.append(string) filename_format = settings.pop("file_name_format", None) if filename_format: settings["filename_format"] = filename_format reformats = { k: v for k, v in settings.items() if "_format" in k } bl = ["date_format"] reformats = { k: v for k, v in reformats.items() if k not in bl } for re_name, re_value in reformats.items(): top = ["{id}", "{file_name}"] bottom = ["{media_id}", "{filename}"] z = list(zip(top, bottom)) for x in z: if x[0] in re_value: settings[re_name] = settings[ re_name].replace(x[0], x[1]) reformats[re_name] = settings[re_name] x = format_types(reformats) q = x.check_rules() if not q[1]: fix.append(f"{key2} - {q[0]}") c = x.check_unique() if not c["bool_status"]: s = f"{key2} - {c['string']}" s_list = s.split("\n") fix.extend(s_list) print value.pop("fourchan", None) value.pop("bbwchan", None) added = "\n".join([f"Added {x}" for x in added if x]) changed = "\n".join([f"Changed {x}" for x in changed if x]) fix = "\n".join([f"Fix: {x}" for x in fix if x]) seperator = "\n" * 2 changed2 = seperator.join([added, changed, fix]) if not all(x for x in changed2.split("\n") if not x): changed2 = changed2.strip() if changed2: print(f"\n{changed2}") if fix: string = "\nFix the problems above and then restart the script." print(string.upper()) input() exit(0) return config, changed2
def fix(config={}): added = [] changed = [] fix = [] for key, value in config.items(): if key == "supported": for key2, value2 in value.items(): for key3, settings in value2.items(): if key3 == "settings": settings["text_length"] = int(settings["text_length"]) re = settings.pop("download_paths", None) if re: settings["download_directories"] = re string = f"download_paths to download_directories in {key2}" changed.append(string) re = settings.get("metadata_directory_format", None) if not re: settings[ "metadata_directory_format"] = "{site_name}/{username}/Metadata" string = f"metadata_directory_format in {key2}" added.append(string) filename_format = settings.pop("file_name_format", None) if filename_format: settings["filename_format"] = filename_format reformats = { k: v for k, v in settings.items() if "_format" in k } bl = ["date_format"] reformats = { k: v for k, v in reformats.items() if k not in bl } for re_name, re_value in reformats.items(): top = ["{id}", "{file_name}"] bottom = ["{media_id}", "{filename}"] z = list(zip(top, bottom)) for x in z: if x[0] in re_value: settings[re_name] = settings[ re_name].replace(x[0], x[1]) reformats[re_name] = settings[re_name] x = format_types(reformats) q = x.check_rules() if not q[1]: fix.append(f"{key2} - {q[0]}") c = x.check_unique() if not c["bool_status"]: s = f"{key2} - {c['string']}" s_list = s.split("\n") fix.extend(s_list) print added = "\n".join([f"Added {x}" for x in added if x]) changed = "\n".join([f"Changed {x}" for x in changed if x]) fix = "\n".join([f"Fix: {x}" for x in fix if x]) seperator = "\n" * 2 changed2 = seperator.join([added, changed, fix]) if not all(x for x in changed2.split("\n") if not x): changed2 = changed2.strip() if changed2: print(f"\n{changed2}") if fix: string = "\nFix the problems above and then restart the script." print(string.upper()) input() exit(0) return config, changed2
async def start( api: onlyfans.start, Session, api_type, site_name, subscription: create_user, folder, json_settings, ): api_table = folder.table_picker(api_type) database_session = Session() # Slow result = database_session.query(api_table).all() metadata = getattr(subscription.temp_scraped, api_type) download_info = subscription.download_info root_directory = download_info["directory"] date_format = json_settings["date_format"] text_length = json_settings["text_length"] reformats = {} reformats["metadata_directory_format"] = json_settings[ "metadata_directory_format"] reformats["file_directory_format"] = json_settings["file_directory_format"] reformats["filename_format"] = json_settings["filename_format"] model_username = subscription.username option = {} option["site_name"] = site_name option["api_type"] = api_type option["profile_username"] = subscription.subscriber.username option["model_username"] = model_username option["date_format"] = date_format option["maximum_length"] = text_length option["directory"] = root_directory formatted = format_types(reformats).check_unique() unique = formatted["unique"] for key, value in reformats.items(): key2 = getattr(unique, key)[0] reformats[key] = value.split(key2, 1)[0] + key2 print print a, base_directory, c = await prepare_reformat( option, keep_vars=True).reformat(reformats) download_info["base_directory"] = base_directory print all_files = [] for root, subdirs, files in os.walk(base_directory): x = [os.path.join(root, x) for x in files] all_files.extend(x) await fix_directories( result, api, subscription, all_files, database_session, folder, site_name, api_type, root_directory, json_settings, ) database_session.close() return metadata