directory=directory_manager.root_metadata_directory, ) p_r.api_type = api_type result: list[api_table] = database_session.query(api_table_).all() metadata = getattr(subscription.temp_scraped, api_type) await fix_directories( result, subscription, database_session, api_type, ) database_session.close() return metadata if __name__ == "__main__": # WORK IN PROGRESS from classes.make_settings import Config from apis.onlyfans import onlyfans as OnlyFans import helpers.main_helper as main_helper config_path = Path(".settings", "config.json") config, _updated = main_helper.get_config(config_path) api_ = OnlyFans.start( config=config, ) subscription = "" input("WIP") exit()
def start_datascraper(json_config: dict, site_name_lower: str, api: Optional[OnlyFans.start] = None, webhooks=True): json_settings = json_config["settings"] json_webhooks = json_settings["webhooks"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_model_choice = json_site_settings["auto_model_choice"] if isinstance(auto_model_choice, str): temp_identifiers = auto_model_choice.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] original_sessions = [] original_sessions = api_helper.create_session( settings=json_settings) original_sessions = [x for x in original_sessions if x] if not original_sessions: print("Unable to create session") return False archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" module = m_onlyfans if not api: api = OnlyFans.start() api = main_helper.process_profiles( json_settings, original_sessions, site_name, api) print subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options( api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option( subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars(auth.auth_details, json_config, json_site_settings, site_name) setup = False setup, subscriptions = module.account_setup( auth, identifiers, jobs, auth_count) if not setup: if webhooks: x = main_helper.process_webhooks( api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.__dict__ profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json") main_helper.export_data( auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions x = main_helper.process_webhooks( api, "auth_webhook", "succeeded") subscription_list = module.format_options( subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(api, identifiers) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name) x = main_helper.process_downloads(api, module) if webhooks: x = main_helper.process_webhooks( api, "download_webhook", "succeeded") elif site_name_lower == "starsavn": pass # site_name = "StarsAVN" # original_api = StarsAVN # module = m_starsavn # apis = main_helper.process_profiles( # json_settings, original_sessions, site_name, original_api) # auto_profile_choice = json_site_settings["auto_profile_choice"] # subscription_array = [] # auth_count = -1 # jobs = json_site_settings["jobs"] # subscription_list = module.format_options( # apis, "users") # apis = choose_option( # subscription_list, auto_profile_choice) # apis = [x.pop(0) for x in apis] # for api in apis: # module.assign_vars(api.auth.auth_details, json_config, # json_site_settings, site_name) # identifier = "" # setup = False # setup = module.account_setup(api, identifier=identifier) # if not setup: # auth_details = api.auth.auth_details.__dict__ # user_auth_filepath = os.path.join( # api.auth.profile_directory, "auth.json") # main_helper.export_data( # auth_details, user_auth_filepath) # continue # if jobs["scrape_names"]: # array = module.manage_subscriptions( # api, auth_count, identifier=identifier) # subscription_array += array # subscription_list = module.format_options( # subscription_array, "usernames") # if jobs["scrape_paid_content"]: # print("Scraping Paid Content") # paid_content = module.paid_content_scraper(apis) # if jobs["scrape_names"]: # print("Scraping Subscriptions") # names = main_helper.process_names( # module, subscription_list, auto_model_choice, apis, json_config, site_name_lower, site_name) # x = main_helper.process_downloads(apis, module) stop_time = str( int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') return api
def start_datascraper(): parser = ArgumentParser() parser.add_argument("-m", "--metadata", action='store_true', help="only exports metadata") args = parser.parse_args() if args.metadata: print("Exporting Metadata Only") log_error = main_helper.setup_logger('errors', 'errors.log') console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s') console.setFormatter(formatter) logging.getLogger("").addHandler(console) # root = os.getcwd() config_path = os.path.join('.settings', 'config.json') json_config, json_config2 = main_helper.get_config(config_path) json_settings = json_config["settings"] json_sites = json_config["supported"] infinite_loop = json_settings["infinite_loop"] global_user_agent = json_settings['global_user_agent'] domain = json_settings["auto_site_choice"] path = os.path.join('.settings', 'extra_auth.json') # extra_auth_config, extra_auth_config2 = main_helper.get_config(path) extra_auth_config = {} exit_on_completion = json_settings['exit_on_completion'] loop_timeout = json_settings['loop_timeout'] main_helper.assign_vars(json_config) string, site_names = module_chooser(domain, json_sites) try: while True: if domain: if site_names: site_name = domain else: print(string) continue else: print(string) x = input() if x == "x": break x = int(x) site_name = site_names[x] site_name_lower = site_name.lower() json_auth_array = [json_sites[site_name_lower]["auth"]] json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] extra_auth_settings = json_sites[site_name_lower][ "extra_auth_settings"] if "extra_auth_settings" in json_sites[ site_name_lower] else { "extra_auth": False } extra_auth = extra_auth_settings["extra_auth"] if extra_auth: choose_auth = extra_auth_settings["choose_auth"] merge_auth = extra_auth_settings["merge_auth"] json_auth_array += extra_auth_config["supported"][ site_name_lower]["auths"] if choose_auth: json_auth_array = main_helper.choose_auth(json_auth_array) apis = [] module = m_onlyfans subscription_array = [] legacy = True original_sessions = api_helper.create_session( settings=json_settings) if not original_sessions: print("Unable to create session") continue archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] for json_auth in json_auth_array: api = OnlyFans.start(original_sessions) auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] module = m_onlyfans module.assign_vars(json_auth, json_config, json_site_settings, site_name) api.set_auth_details(**json_auth, global_user_agent=user_agent) identifier = "" setup = module.account_setup(api, identifier=identifier) if not setup: continue if jobs["scrape_names"]: array = module.manage_subscriptions( api, auth_count, identifier=identifier) subscription_array += array apis.append(api) subscription_list = module.format_options( subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis) if jobs["scrape_names"]: print("Scraping Subscriptions") x = main_helper.process_names(module, subscription_list, auto_scrape_names, json_auth_array, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) print elif site_name_lower == "starsavn": site_name = "StarsAVN" subscription_array = [] auth_count = -1 for json_auth in json_auth_array: sessions = api_helper.copy_sessions(original_sessions) api = StarsAVN.start(sessions) auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] module = m_starsavn module.assign_vars(json_auth, json_config, json_site_settings, site_name) api.set_auth_details(**json_auth, global_user_agent=user_agent) setup = module.account_setup(api) if not setup: continue jobs = json_site_settings["jobs"] if jobs["scrape_names"]: array = module.manage_subscriptions(api, auth_count) subscription_array += array if jobs["scrape_paid_content"]: paid_contents = api.get_paid_content() paid_content = module.paid_content_scraper(api) apis.append(api) subscription_array = module.format_options( subscription_array, "usernames") stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') if exit_on_completion: print("Now exiting.") exit(0) elif not infinite_loop: print("Input anything to continue") input() elif loop_timeout: print('Pausing scraper for ' + loop_timeout + ' seconds.') time.sleep(int(loop_timeout)) except Exception as e: log_error.exception(e) input()
async def start_datascraper( config: Config, site_name: str, api_: Optional[OnlyFans.start | Fansly.start | StarsAVN.start] = None, webhooks: bool = True, ): global_settings = config.settings proxies: list[str] = await api_helper.test_proxies(global_settings.proxies) if global_settings.proxies and not proxies: print("Unable to create session") return None async def default( datascraper: Optional[m_onlyfans.OnlyFansDataScraper] | Optional[m_fansly.FanslyDataScraper] | Optional[m_starsavn.StarsAVNDataScraper], ): if not datascraper: return api = datascraper.api global_settings = api.get_global_settings() site_settings = api.get_site_settings() if not (global_settings and site_settings): return await main_helper.process_profiles(api, global_settings) subscription_array: list[user_types] = [] auth_count = 0 profile_options = OptionsFormat( api.auths, "profiles", site_settings.auto_profile_choice ) api.auths = profile_options.final_choices identifiers = [] if site_settings.auto_model_choice: subscription_options = OptionsFormat( subscription_array, "subscriptions", site_settings.auto_model_choice ) if not subscription_options.scrape_all(): identifiers = subscription_options.choice_list for auth in api.auths: auth: auth_types = auth if not auth.auth_details: continue setup = False setup, subscriptions = await account_setup( auth, datascraper, site_settings, identifiers ) if not setup: if webhooks: await main_helper.process_webhooks( api, "auth_webhook", "failed", global_settings ) auth_details: dict[str, Any] = {} auth_details["auth"] = auth.auth_details.export() profile_directory = api.base_directory_manager.profile.root_directory user_auth_filepath = profile_directory.joinpath( api.site_name, auth.auth_details.username, "auth.json" ) main_helper.export_json(auth_details, user_auth_filepath) continue auth_count += 1 subscription_array.extend(subscriptions) await main_helper.process_webhooks( api, "auth_webhook", "succeeded", global_settings ) # Do stuff with authed user subscription_options = OptionsFormat( subscription_array, "subscriptions", site_settings.auto_model_choice ) datascraper.subscription_options = subscription_options subscription_list = subscription_options.final_choices await main_helper.process_jobs(datascraper, subscription_list, site_settings) await main_helper.process_downloads(api, datascraper, global_settings) if webhooks: await main_helper.process_webhooks( api, "download_webhook", "succeeded", global_settings ) archive_time = timeit.default_timer() datascraper = None match site_name: case "OnlyFans": if not isinstance(api_, OnlyFans.start): api_ = OnlyFans.start( config=config, ) datascraper = m_onlyfans.OnlyFansDataScraper(api_) case "Fansly": if not isinstance(api_, Fansly.start): api_ = Fansly.start( config=config, ) datascraper = m_fansly.FanslyDataScraper(api_) case "StarsAVN": if not isinstance(api_, StarsAVN.start): api_ = StarsAVN.start( config=config, ) datascraper = m_starsavn.StarsAVNDataScraper(api_) case _: pass await default(datascraper) stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print("Archive Completed in " + stop_time + " Minutes") return api_
async def start_datascraper( json_config: dict[Any,Any], site_name_lower: str, api: Optional[OnlyFans.start| Fansly.start] = None, webhooks:bool=True, ) -> Optional[OnlyFans.start]: json_settings = json_config["settings"] json_webhooks = json_settings["webhooks"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_model_choice = json_site_settings["auto_model_choice"] if isinstance(auto_model_choice, str): temp_identifiers = auto_model_choice.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] proxies:list[str] = await api_helper.test_proxies(json_settings["proxies"]) if json_settings["proxies"] and not proxies: print("Unable to create session") return None archive_time = timeit.default_timer() match site_name_lower: case "onlyfans": site_name = "OnlyFans" module = m_onlyfans if not api: api = OnlyFans.start(max_threads=json_settings["max_threads"]) api.settings = json_config api = main_helper.process_profiles(json_settings, proxies, site_name, api) print subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options(api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option(subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars( auth.auth_details, json_config, json_site_settings, site_name ) setup = False setup, subscriptions = await module.account_setup( auth, identifiers, jobs, auth_count ) if not setup: if webhooks: await main_helper.process_webhooks(api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.export() profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json" ) main_helper.export_data(auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions await main_helper.process_webhooks(api, "auth_webhook", "succeeded") # Do stuff with authed user subscription_list = module.format_options( subscription_array, "usernames", api.auths ) if jobs["scrape_paid_content"] and api.has_active_auths(): print("Scraping Paid Content") await module.paid_content_scraper(api, identifiers) if jobs["scrape_names"] and api.has_active_auths(): print("Scraping Subscriptions") await main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name, ) await main_helper.process_downloads(api, module) if webhooks: await main_helper.process_webhooks(api, "download_webhook", "succeeded") case "fansly": site_name = "Fansly" module = m_fansly if not api: api = Fansly.start(max_threads=json_settings["max_threads"]) api.settings = json_config api = main_helper.process_profiles(json_settings, proxies, site_name, api) subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options(api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option(subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars( auth.auth_details, json_config, json_site_settings, site_name ) setup = False setup, subscriptions = await module.account_setup( auth, identifiers, jobs, auth_count ) if not setup: if webhooks: await main_helper.process_webhooks(api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.export() profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json" ) main_helper.export_data(auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions await main_helper.process_webhooks(api, "auth_webhook", "succeeded") # Do stuff with authed user subscription_list = module.format_options( subscription_array, "usernames", api.auths ) if jobs["scrape_paid_content"] and api.has_active_auths(): print("Scraping Paid Content") await module.paid_content_scraper(api, identifiers) if jobs["scrape_names"] and api.has_active_auths(): print("Scraping Subscriptions") await main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name, ) await main_helper.process_downloads(api, module) if webhooks: await main_helper.process_webhooks(api, "download_webhook", "succeeded") case "starsavn": pass # site_name = "StarsAVN" # original_api = StarsAVN # module = m_starsavn # apis = main_helper.process_profiles( # json_settings, original_sessions, site_name, original_api) # auto_profile_choice = json_site_settings["auto_profile_choice"] # subscription_array = [] # auth_count = -1 # jobs = json_site_settings["jobs"] # subscription_list = module.format_options( # apis, "users") # apis = choose_option( # subscription_list, auto_profile_choice) # apis = [x.pop(0) for x in apis] # for api in apis: # module.assign_vars(api.auth.auth_details, json_config, # json_site_settings, site_name) # identifier = "" # setup = False # setup = module.account_setup(api, identifier=identifier) # if not setup: # auth_details = api.auth.auth_details.__dict__ # user_auth_filepath = os.path.join( # api.auth.profile_directory, "auth.json") # main_helper.export_data( # auth_details, user_auth_filepath) # continue # if jobs["scrape_names"]: # array = module.manage_subscriptions( # api, auth_count, identifier=identifier) # subscription_array += array # subscription_list = module.format_options( # subscription_array, "usernames") # if jobs["scrape_paid_content"]: # print("Scraping Paid Content") # paid_content = module.paid_content_scraper(apis) # if jobs["scrape_names"]: # print("Scraping Subscriptions") # names = main_helper.process_names( # module, subscription_list, auto_model_choice, apis, json_config, site_name_lower, site_name) # x = main_helper.process_downloads(apis, module) stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print("Archive Completed in " + stop_time + " Minutes") return api