def start(subscription, api_type, api_path, site_name, json_settings):
metadata = getattr(subscription.scraped, api_type)
download_info = subscription.download_info
root_directory = download_info["directory"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
reformats = {}
reformats["metadata_directory_format"] = json_settings[
"metadata_directory_format"]
reformats["file_directory_format"] = json_settings["file_directory_format"]
reformats["filename_format"] = json_settings["filename_format"]
username = subscription.username
option = {}
option["site_name"] = site_name
option["api_type"] = api_type
option["username"] = username
option["date_format"] = date_format
option["maximum_length"] = text_length
option["directory"] = root_directory
formatted = format_types(reformats).check_unique()
unique = formatted["unique"]
for key, value in reformats.items():
key2 = getattr(unique, key)[0]
reformats[key] = value.split(key2, 1)[0] + key2
print
print
a, base_directory, c = prepare_reformat(option,
keep_vars=True).reformat(reformats)
download_info["base_directory"] = base_directory
print
all_files = []
for root, subdirs, files in os.walk(base_directory):
x = [os.path.join(root, x) for x in files]
all_files.extend(x)
for media_type, value in metadata.content:
if media_type == "Texts":
continue
for status, value2 in value:
fixed, new_directories = fix_directories(value2, root_directory,
site_name, api_path,
media_type, username,
all_files, json_settings)
for new_directory in new_directories:
directory = os.path.abspath(new_directory)
os.makedirs(directory, exist_ok=True)
fixed2 = fix_metadata(fixed)
setattr(value, status, fixed2)
setattr(
metadata.content,
media_type,
value,
)
return metadata
def start(Session, parent_type, api_type, api_path, site_name, subscription,
folder, json_settings):
api_table = folder.api_table
media_table = folder.media_table
database_session = Session()
result = database_session.query(api_table).all()
metadata = getattr(subscription.scraped, api_type)
download_info = subscription.download_info
root_directory = download_info["directory"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
reformats = {}
reformats["metadata_directory_format"] = json_settings[
"metadata_directory_format"]
reformats["file_directory_format"] = json_settings["file_directory_format"]
reformats["filename_format"] = json_settings["filename_format"]
username = subscription.username
option = {}
option["site_name"] = site_name
option["api_type"] = api_type
option["username"] = username
option["date_format"] = date_format
option["maximum_length"] = text_length
option["directory"] = root_directory
formatted = format_types(reformats).check_unique()
unique = formatted["unique"]
for key, value in reformats.items():
key2 = getattr(unique, key)[0]
reformats[key] = value.split(key2, 1)[0] + key2
print
print
a, base_directory, c = prepare_reformat(option,
keep_vars=True).reformat(reformats)
download_info["base_directory"] = base_directory
print
all_files = []
for root, subdirs, files in os.walk(base_directory):
x = [os.path.join(root, x) for x in files]
all_files.extend(x)
fixed, new_directories = fix_directories(result, all_files,
database_session, folder,
site_name, parent_type, api_type,
username, root_directory,
json_settings)
database_session.close()
return metadata
def fix(config={}):
added = []
changed = []
fix = []
settings = {}
global_user_agent = ""
for key, value in config.items():
if key == "settings":
settings = value
auto_profile_choice = settings.pop("auto_profile_choice", None)
socks5_proxies = settings.pop("socks5_proxy", None)
if socks5_proxies:
fixed_socks5_proxies = []
for socks5_proxy in socks5_proxies:
fixed_socks5_proxy = f"socks5://{socks5_proxy}"
fixed_socks5_proxies.append(fixed_socks5_proxy)
settings["proxies"] = fixed_socks5_proxies
global_user_agent = settings.pop("global_user_agent", None)
if key == "supported":
for key2, value2 in value.items():
temp_auth = value2.pop("auth", None)
if temp_auth:
q = os.path.abspath(".settings")
backup_config_filepath = os.path.join(
q, "config_backup.json")
print(
f"LEGACY CONFIG FOUND, BACKING IT UP AND CREATING A NEW ONE. ({backup_config_filepath})"
)
export_json(backup_config_filepath, config)
print
temp_auth["user_agent"] = global_user_agent
auth = {}
temp_auth = auth_details(temp_auth).__dict__
auth["auth"] = temp_auth
if "profile_directories" in settings:
dpd = settings["profile_directories"][0]
default_profile_directory = os.path.join(
os.path.abspath(dpd), key2, "default")
os.makedirs(default_profile_directory, exist_ok=True)
profile_auth_filepath = os.path.join(
default_profile_directory, "auth.json")
export_json(profile_auth_filepath, auth)
print(
f"{profile_auth_filepath} HAS BEEN CREATED, CHECK IF IT'S CORRECT."
)
print
for key3, settings in value2.items():
if key3 == "settings":
settings["text_length"] = int(settings["text_length"])
re = settings.pop("download_paths", None)
if re:
settings["download_directories"] = re
string = f"download_paths to download_directories in {key2}"
changed.append(string)
re = settings.get("metadata_directory_format", None)
if not re:
settings[
"metadata_directory_format"] = "{site_name}/{username}/Metadata"
string = f"metadata_directory_format in {key2}"
added.append(string)
filename_format = settings.pop("file_name_format",
None)
if filename_format:
settings["filename_format"] = filename_format
reformats = {
k: v
for k, v in settings.items() if "_format" in k
}
bl = ["date_format"]
reformats = {
k: v
for k, v in reformats.items() if k not in bl
}
for re_name, re_value in reformats.items():
top = ["{id}", "{file_name}"]
bottom = ["{media_id}", "{filename}"]
z = list(zip(top, bottom))
for x in z:
if x[0] in re_value:
settings[re_name] = settings[
re_name].replace(x[0], x[1])
reformats[re_name] = settings[re_name]
x = format_types(reformats)
q = x.check_rules()
if not q[1]:
fix.append(f"{key2} - {q[0]}")
c = x.check_unique()
if not c["bool_status"]:
s = f"{key2} - {c['string']}"
s_list = s.split("\n")
fix.extend(s_list)
print
value.pop("fourchan", None)
value.pop("bbwchan", None)
added = "\n".join([f"Added {x}" for x in added if x])
changed = "\n".join([f"Changed {x}" for x in changed if x])
fix = "\n".join([f"Fix: {x}" for x in fix if x])
seperator = "\n" * 2
changed2 = seperator.join([added, changed, fix])
if not all(x for x in changed2.split("\n") if not x):
changed2 = changed2.strip()
if changed2:
print(f"\n{changed2}")
if fix:
string = "\nFix the problems above and then restart the script."
print(string.upper())
input()
exit(0)
return config, changed2
def fix(config={}):
added = []
changed = []
fix = []
for key, value in config.items():
if key == "supported":
for key2, value2 in value.items():
for key3, settings in value2.items():
if key3 == "settings":
settings["text_length"] = int(settings["text_length"])
re = settings.pop("download_paths", None)
if re:
settings["download_directories"] = re
string = f"download_paths to download_directories in {key2}"
changed.append(string)
re = settings.get("metadata_directory_format", None)
if not re:
settings[
"metadata_directory_format"] = "{site_name}/{username}/Metadata"
string = f"metadata_directory_format in {key2}"
added.append(string)
filename_format = settings.pop("file_name_format",
None)
if filename_format:
settings["filename_format"] = filename_format
reformats = {
k: v
for k, v in settings.items() if "_format" in k
}
bl = ["date_format"]
reformats = {
k: v
for k, v in reformats.items() if k not in bl
}
for re_name, re_value in reformats.items():
top = ["{id}", "{file_name}"]
bottom = ["{media_id}", "{filename}"]
z = list(zip(top, bottom))
for x in z:
if x[0] in re_value:
settings[re_name] = settings[
re_name].replace(x[0], x[1])
reformats[re_name] = settings[re_name]
x = format_types(reformats)
q = x.check_rules()
if not q[1]:
fix.append(f"{key2} - {q[0]}")
c = x.check_unique()
if not c["bool_status"]:
s = f"{key2} - {c['string']}"
s_list = s.split("\n")
fix.extend(s_list)
print
added = "\n".join([f"Added {x}" for x in added if x])
changed = "\n".join([f"Changed {x}" for x in changed if x])
fix = "\n".join([f"Fix: {x}" for x in fix if x])
seperator = "\n" * 2
changed2 = seperator.join([added, changed, fix])
if not all(x for x in changed2.split("\n") if not x):
changed2 = changed2.strip()
if changed2:
print(f"\n{changed2}")
if fix:
string = "\nFix the problems above and then restart the script."
print(string.upper())
input()
exit(0)
return config, changed2
async def start(
api: onlyfans.start,
Session,
api_type,
site_name,
subscription: create_user,
folder,
json_settings,
):
api_table = folder.table_picker(api_type)
database_session = Session()
# Slow
result = database_session.query(api_table).all()
metadata = getattr(subscription.temp_scraped, api_type)
download_info = subscription.download_info
root_directory = download_info["directory"]
date_format = json_settings["date_format"]
text_length = json_settings["text_length"]
reformats = {}
reformats["metadata_directory_format"] = json_settings[
"metadata_directory_format"]
reformats["file_directory_format"] = json_settings["file_directory_format"]
reformats["filename_format"] = json_settings["filename_format"]
model_username = subscription.username
option = {}
option["site_name"] = site_name
option["api_type"] = api_type
option["profile_username"] = subscription.subscriber.username
option["model_username"] = model_username
option["date_format"] = date_format
option["maximum_length"] = text_length
option["directory"] = root_directory
formatted = format_types(reformats).check_unique()
unique = formatted["unique"]
for key, value in reformats.items():
key2 = getattr(unique, key)[0]
reformats[key] = value.split(key2, 1)[0] + key2
print
print
a, base_directory, c = await prepare_reformat(
option, keep_vars=True).reformat(reformats)
download_info["base_directory"] = base_directory
print
all_files = []
for root, subdirs, files in os.walk(base_directory):
x = [os.path.join(root, x) for x in files]
all_files.extend(x)
await fix_directories(
result,
api,
subscription,
all_files,
database_session,
folder,
site_name,
api_type,
root_directory,
json_settings,
)
database_session.close()
return metadata
