def start_datascraper(): parser = ArgumentParser() parser.add_argument("-m", "--metadata", action='store_true', help="only exports metadata") args = parser.parse_args() if args.metadata: print("Exporting Metadata Only") log_error = main_helper.setup_logger('errors', 'errors.log') console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s') console.setFormatter(formatter) logging.getLogger("").addHandler(console) # root = os.getcwd() config_path = os.path.join('.settings', 'config.json') json_config, json_config2 = main_helper.get_config(config_path) json_settings = json_config["settings"] json_sites = json_config["supported"] infinite_loop = json_settings["infinite_loop"] global_user_agent = json_settings['global_user_agent'] domain = json_settings["auto_site_choice"] path = os.path.join('.settings', 'extra_auth.json') extra_auth_config = json.load(open(path)) exit_on_completion = json_settings['exit_on_completion'] loop_timeout = json_settings['loop_timeout'] string = "Site: " site_names = [] bl = ["patreon"] if not domain: site_count = len(json_sites) count = 0 for x in json_sites: if x in bl: continue string += str(count) + " = " + x site_names.append(x) if count + 1 != site_count: string += " | " count += 1 string += "x = Exit" try: while True: if domain: site_name = domain else: print(string) x = input() if x == "x": break x = int(x) site_name = site_names[x] site_name_lower = site_name.lower() json_auth_array = [json_sites[site_name_lower]["auth"]] json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] extra_auth_settings = json_sites[site_name_lower][ "extra_auth_settings"] if "extra_auth_settings" in json_sites[ site_name_lower] else { "extra_auth": False } extra_auth = extra_auth_settings["extra_auth"] if extra_auth: choose_auth = extra_auth_settings["choose_auth"] merge_auth = extra_auth_settings["merge_auth"] json_auth_array += extra_auth_config[site_name_lower][ "extra_auth"] if choose_auth: json_auth_array = main_helper.choose_auth(json_auth_array) session_array = [] x = onlyfans app_token = "" subscription_array = [] legacy = True if site_name_lower == "onlyfans": legacy = False site_name = "OnlyFans" subscription_array = [] auth_count = -1 x.assign_vars(json_config, json_site_settings, site_name) for json_auth in json_auth_array: auth_count += 1 app_token = json_auth['app_token'] user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] x = onlyfans session = x.create_session() if not session: print("Unable to create session") continue session = x.create_auth(session, user_agent, app_token, json_auth) session_array.append(session) if not session["session"]: continue # x.get_paid_posts(session["session"],app_token) cookies = session["session"].cookies.get_dict() auth_id = cookies["auth_id"] json_auth['auth_id'] = auth_id json_auth['auth_uniq_'] = cookies["auth_uniq_" + auth_id] json_auth['auth_hash'] = cookies["auth_hash"] json_auth['sess'] = cookies["sess"] json_auth['fp'] = cookies["fp"] if json_config != json_config2: update_config(json_config) me_api = session["me_api"] array = x.get_subscriptions(session["session"], app_token, session["subscriber_count"], me_api, auth_count) subscription_array += array subscription_array = x.format_options(subscription_array, "usernames") if site_name_lower == "patreon": legacy = False site_name = "Patreon" subscription_array = [] auth_count = -1 x = patreon x.assign_vars(json_config, json_site_settings, site_name) for json_auth in json_auth_array: auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] session = x.create_session() session = x.create_auth(session, user_agent, json_auth) session_array.append(session) if not session["session"]: continue cookies = session["session"].cookies.get_dict() json_auth['session_id'] = cookies["session_id"] if json_config != json_config2: update_config(json_config) me_api = session["me_api"] array = x.get_subscriptions(session["session"], auth_count) subscription_array += array subscription_array = x.format_options(subscription_array, "usernames") elif site_name_lower == "starsavn": legacy = False site_name = "StarsAVN" subscription_array = [] auth_count = -1 x = starsavn x.assign_vars(json_config, json_site_settings, site_name) for json_auth in json_auth_array: auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] sess = json_auth['sess'] auth_array = dict() auth_array["sess"] = sess session = x.create_session() session = x.create_auth(session, user_agent, app_token, json_auth) session_array.append(session) if not session["session"]: continue me_api = session["me_api"] array = x.get_subscriptions(session["session"], app_token, session["subscriber_count"], me_api, auth_count) subscription_array += array subscription_array = x.format_options(subscription_array, "usernames") elif site_name == "fourchan": x = fourchan site_name = "4Chan" x.assign_vars(json_config, json_site_settings, site_name) session_array = [x.create_session()] array = x.get_subscriptions() subscription_array = x.format_options(array) elif site_name == "bbwchan": x = bbwchan site_name = "BBWChan" x.assign_vars(json_config, json_site_settings, site_name) session_array = [x.create_session()] array = x.get_subscriptions() subscription_array = x.format_options(array) names = subscription_array[0] if names: print("Names: Username = username | " + subscription_array[1]) if not auto_scrape_names: value = input().strip() if value.isdigit(): if value == "0": names = names[1:] else: names = [names[int(value)]] else: names = [name for name in names if value in name[1]] else: value = 0 names = names[1:] else: print("There's nothing to scrape.") continue start_time = timeit.default_timer() download_list = [] for name in names: # Extra Auth Support if not legacy: json_auth = json_auth_array[name[0]] auth_count = name[0] session = session_array[auth_count]["session"] name = name[-1] else: session = session_array[0]["session"] main_helper.assign_vars(json_config) username = main_helper.parse_links(site_name_lower, name) result = x.start_datascraper(session, username, site_name, app_token, choice_type=value) if not args.metadata: download_list.append(result) for y in download_list: for arg in y[1]: x.download_media(*arg) stop_time = str(int(timeit.default_timer() - start_time) / 60) print('Task Completed in ' + stop_time + ' Minutes') if exit_on_completion: print("Now exiting.") exit(0) elif not infinite_loop: print("Input anything to continue") input() elif loop_timeout: print('Pausing scraper for ' + loop_timeout + ' seconds.') time.sleep(int(loop_timeout)) except Exception as e: log_error.exception(e) input()
def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=True): json_settings = json_config["settings"] json_webhooks = json_settings["webhooks"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] if isinstance(auto_scrape_names, str): temp_identifiers = auto_scrape_names.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] original_sessions = [] original_sessions = api_helper.create_session(settings=json_settings) original_sessions = [x for x in original_sessions if x] if not original_sessions: print("Unable to create session") return False archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" original_api = OnlyFans module = m_onlyfans if not apis: session_manager = api_helper.session_manager() apis = main_helper.process_profiles(json_settings, session_manager, site_name, original_api) subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] subscription_list = module.format_options(apis, "users") if not auto_profile_choice: print("Choose Profile") apis = choose_option(subscription_list, auto_profile_choice) apis = [x.pop(0) for x in apis] for api in apis: api.session_manager.copy_sessions(original_sessions) module.assign_vars(api.auth.auth_details, json_config, json_site_settings, site_name) setup = False setup, subscriptions = module.account_setup(api, identifiers, jobs) if not setup: if webhooks: x = main_helper.process_webhooks([api], "auth_webhook", "failed") auth_details = {} auth_details["auth"] = api.auth.auth_details.__dict__ profile_directory = api.auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( api.auth.profile_directory, "auth.json") main_helper.export_data(auth_details, user_auth_filepath) continue subscription_array += subscriptions x = main_helper.process_webhooks([api], "auth_webhook", "succeeded") subscription_list = module.format_options(subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis, identifiers) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names(module, subscription_list, auto_scrape_names, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) if webhooks: x = main_helper.process_webhooks(apis, "download_webhook", "succeeded") elif site_name_lower == "starsavn": site_name = "StarsAVN" original_api = StarsAVN module = m_starsavn apis = main_helper.process_profiles(json_settings, original_sessions, site_name, original_api) auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] subscription_list = module.format_options(apis, "users") apis = choose_option(subscription_list, auto_profile_choice) apis = [x.pop(0) for x in apis] for api in apis: module.assign_vars(api.auth.auth_details, json_config, json_site_settings, site_name) identifier = "" setup = False setup = module.account_setup(api, identifier=identifier) if not setup: auth_details = api.auth.auth_details.__dict__ user_auth_filepath = os.path.join(api.auth.profile_directory, "auth.json") main_helper.export_data(auth_details, user_auth_filepath) continue if jobs["scrape_names"]: array = module.manage_subscriptions(api, auth_count, identifier=identifier) subscription_array += array subscription_list = module.format_options(subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names(module, subscription_list, auto_scrape_names, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') return apis
def start_datascraper(): parser = ArgumentParser() parser.add_argument("-m", "--metadata", action='store_true', help="only exports metadata") args = parser.parse_args() if args.metadata: print("Exporting Metadata Only") log_error = main_helper.setup_logger('errors', 'errors.log') console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s') console.setFormatter(formatter) logging.getLogger("").addHandler(console) # Open config.json and fill in MANDATORY information for the script to work path = os.path.join('.settings', 'config.json') if os.path.isfile(path): json_config = json.load(open(path)) else: json_config = {} json_config2 = json.loads( json.dumps(make_config.start(**json_config), default=lambda o: o.__dict__)) if json_config != json_config2: update_config(json_config2) if not json_config: input( "The .settings\\config.json file has been created. Fill in whatever you need to fill in and then press enter when done.\n" ) json_config2 = json.load(open(path)) json_config = copy.deepcopy(json_config2) json_settings = json_config["settings"] json_sites = json_config["supported"] infinite_loop = json_settings["infinite_loop"] global_user_agent = json_settings['global_user_agent'] domain = json_settings["auto_site_choice"] path = os.path.join('.settings', 'extra_auth.json') extra_auth_config = json.load(open(path)) exit_on_completion = json_settings['exit_on_completion'] loop_timeout = json_settings['loop_timeout'] string = "" site_names = [] if not domain: site_count = len(json_sites) count = 0 for x in json_sites: string += str(count) + " = " + x site_names.append(x) if count + 1 != site_count: string += " | " count += 1 try: while True: if domain: site_name = domain else: print("Site: " + string) x = int(input()) site_name = site_names[x] site_name_lower = site_name.lower() json_auth_array = [json_sites[site_name_lower]["auth"]] json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] extra_auth_settings = json_sites[site_name_lower][ "extra_auth_settings"] if "extra_auth_settings" in json_sites[ site_name_lower] else { "extra_auth": False } extra_auth = extra_auth_settings["extra_auth"] if extra_auth: choose_auth = extra_auth_settings["choose_auth"] merge_auth = extra_auth_settings["merge_auth"] json_auth_array += extra_auth_config[site_name_lower][ "extra_auth"] if choose_auth: json_auth_array = main_helper.choose_auth(json_auth_array) session_array = [] x = onlyfans app_token = "" subscription_array = [] legacy = True if site_name_lower == "onlyfans": legacy = False site_name = "OnlyFans" subscription_array = [] auth_count = -1 x.assign_vars(json_config, json_site_settings) for json_auth in json_auth_array: auth_count += 1 app_token = json_auth['app_token'] user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] x = onlyfans session = x.create_session() session = x.create_auth(session, user_agent, app_token, json_auth) session_array.append(session) if not session["session"]: continue cookies = session["session"].cookies.get_dict() json_auth['auth_id'] = cookies["auth_id"] json_auth['auth_hash'] = cookies["auth_hash"] json_auth['sess'] = cookies["sess"] json_auth['fp'] = cookies["fp"] if json_config != json_config2: update_config(json_config) me_api = session["me_api"] array = x.get_subscriptions(session["session"], app_token, session["subscriber_count"], me_api, auth_count) subscription_array += array subscription_array = x.format_options(subscription_array, "usernames") elif site_name_lower == "starsavn": legacy = False site_name = "StarsAVN" subscription_array = [] auth_count = -1 x = starsavn x.assign_vars(json_config, json_site_settings) for json_auth in json_auth_array: auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] sess = json_auth['sess'] auth_array = dict() auth_array["sess"] = sess session = x.create_session(user_agent, app_token, auth_array) session_array.append(session) if not session["session"]: continue me_api = session["me_api"] array = x.get_subscriptions(session["session"], app_token, session["subscriber_count"], me_api, auth_count) subscription_array += array subscription_array = x.format_options(subscription_array, "usernames") elif site_name == "fourchan": x = fourchan site_name = "4Chan" x.assign_vars(json_config, json_site_settings) session_array = [x.create_session()] array = x.get_subscriptions() subscription_array = x.format_options(array) elif site_name == "bbwchan": x = bbwchan site_name = "BBWChan" x.assign_vars(json_config, json_site_settings) session_array = [x.create_session()] array = x.get_subscriptions() subscription_array = x.format_options(array) names = subscription_array[0] if names: print("Names: " + subscription_array[1]) if not auto_scrape_names: value = int(input().strip()) else: value = 0 if value: names = [names[value]] else: names.pop(0) else: print("There's nothing to scrape.") continue start_time = timeit.default_timer() download_list = [] for name in names: # Extra Auth Support if not legacy: json_auth = json_auth_array[name[0]] auth_count = name[0] session = session_array[auth_count]["session"] name = name[1] else: session = session_array[0]["session"] main_helper.assign_vars(json_config) username = main_helper.parse_links(site_name_lower, name) result = x.start_datascraper(session, username, site_name, app_token, choice_type=value) if not args.metadata: download_list.append(result) for y in download_list: for arg in y[1]: x.download_media(*arg) stop_time = str(int(timeit.default_timer() - start_time) / 60) print('Task Completed in ' + stop_time + ' Minutes') if exit_on_completion: print("Now exiting.") exit(0) elif not infinite_loop: print("Input anything to continue") input() elif loop_timeout: print('Pausing scraper for ' + loop_timeout + ' seconds.') time.sleep(int(loop_timeout)) except Exception as e: log_error.exception(e) input()
def start_datascraper(json_config: dict, site_name_lower: str, api: Optional[OnlyFans.start] = None, webhooks=True): json_settings = json_config["settings"] json_webhooks = json_settings["webhooks"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_model_choice = json_site_settings["auto_model_choice"] if isinstance(auto_model_choice, str): temp_identifiers = auto_model_choice.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] original_sessions = [] original_sessions = api_helper.create_session( settings=json_settings) original_sessions = [x for x in original_sessions if x] if not original_sessions: print("Unable to create session") return False archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" module = m_onlyfans if not api: api = OnlyFans.start() api = main_helper.process_profiles( json_settings, original_sessions, site_name, api) print subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options( api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option( subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars(auth.auth_details, json_config, json_site_settings, site_name) setup = False setup, subscriptions = module.account_setup( auth, identifiers, jobs, auth_count) if not setup: if webhooks: x = main_helper.process_webhooks( api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.__dict__ profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json") main_helper.export_data( auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions x = main_helper.process_webhooks( api, "auth_webhook", "succeeded") subscription_list = module.format_options( subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(api, identifiers) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name) x = main_helper.process_downloads(api, module) if webhooks: x = main_helper.process_webhooks( api, "download_webhook", "succeeded") elif site_name_lower == "starsavn": pass # site_name = "StarsAVN" # original_api = StarsAVN # module = m_starsavn # apis = main_helper.process_profiles( # json_settings, original_sessions, site_name, original_api) # auto_profile_choice = json_site_settings["auto_profile_choice"] # subscription_array = [] # auth_count = -1 # jobs = json_site_settings["jobs"] # subscription_list = module.format_options( # apis, "users") # apis = choose_option( # subscription_list, auto_profile_choice) # apis = [x.pop(0) for x in apis] # for api in apis: # module.assign_vars(api.auth.auth_details, json_config, # json_site_settings, site_name) # identifier = "" # setup = False # setup = module.account_setup(api, identifier=identifier) # if not setup: # auth_details = api.auth.auth_details.__dict__ # user_auth_filepath = os.path.join( # api.auth.profile_directory, "auth.json") # main_helper.export_data( # auth_details, user_auth_filepath) # continue # if jobs["scrape_names"]: # array = module.manage_subscriptions( # api, auth_count, identifier=identifier) # subscription_array += array # subscription_list = module.format_options( # subscription_array, "usernames") # if jobs["scrape_paid_content"]: # print("Scraping Paid Content") # paid_content = module.paid_content_scraper(apis) # if jobs["scrape_names"]: # print("Scraping Subscriptions") # names = main_helper.process_names( # module, subscription_list, auto_model_choice, apis, json_config, site_name_lower, site_name) # x = main_helper.process_downloads(apis, module) stop_time = str( int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') return api
def start_datascraper(): parser = ArgumentParser() parser.add_argument("-m", "--metadata", action='store_true', help="only exports metadata") args = parser.parse_args() if args.metadata: print("Exporting Metadata Only") log_error = main_helper.setup_logger('errors', 'errors.log') console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s') console.setFormatter(formatter) logging.getLogger("").addHandler(console) # root = os.getcwd() config_path = os.path.join('.settings', 'config.json') json_config, json_config2 = main_helper.get_config(config_path) json_settings = json_config["settings"] json_sites = json_config["supported"] infinite_loop = json_settings["infinite_loop"] global_user_agent = json_settings['global_user_agent'] domain = json_settings["auto_site_choice"] path = os.path.join('.settings', 'extra_auth.json') # extra_auth_config, extra_auth_config2 = main_helper.get_config(path) extra_auth_config = {} exit_on_completion = json_settings['exit_on_completion'] loop_timeout = json_settings['loop_timeout'] main_helper.assign_vars(json_config) string, site_names = module_chooser(domain, json_sites) try: while True: if domain: if site_names: site_name = domain else: print(string) continue else: print(string) x = input() if x == "x": break x = int(x) site_name = site_names[x] site_name_lower = site_name.lower() json_auth_array = [json_sites[site_name_lower]["auth"]] json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] extra_auth_settings = json_sites[site_name_lower][ "extra_auth_settings"] if "extra_auth_settings" in json_sites[ site_name_lower] else { "extra_auth": False } extra_auth = extra_auth_settings["extra_auth"] if extra_auth: choose_auth = extra_auth_settings["choose_auth"] merge_auth = extra_auth_settings["merge_auth"] json_auth_array += extra_auth_config["supported"][ site_name_lower]["auths"] if choose_auth: json_auth_array = main_helper.choose_auth(json_auth_array) apis = [] module = m_onlyfans subscription_array = [] legacy = True original_sessions = api_helper.create_session( settings=json_settings) if not original_sessions: print("Unable to create session") continue archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] for json_auth in json_auth_array: api = OnlyFans.start(original_sessions) auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] module = m_onlyfans module.assign_vars(json_auth, json_config, json_site_settings, site_name) api.set_auth_details(**json_auth, global_user_agent=user_agent) identifier = "" setup = module.account_setup(api, identifier=identifier) if not setup: continue if jobs["scrape_names"]: array = module.manage_subscriptions( api, auth_count, identifier=identifier) subscription_array += array apis.append(api) subscription_list = module.format_options( subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis) if jobs["scrape_names"]: print("Scraping Subscriptions") x = main_helper.process_names(module, subscription_list, auto_scrape_names, json_auth_array, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) print elif site_name_lower == "starsavn": site_name = "StarsAVN" subscription_array = [] auth_count = -1 for json_auth in json_auth_array: sessions = api_helper.copy_sessions(original_sessions) api = StarsAVN.start(sessions) auth_count += 1 user_agent = global_user_agent if not json_auth[ 'user_agent'] else json_auth['user_agent'] module = m_starsavn module.assign_vars(json_auth, json_config, json_site_settings, site_name) api.set_auth_details(**json_auth, global_user_agent=user_agent) setup = module.account_setup(api) if not setup: continue jobs = json_site_settings["jobs"] if jobs["scrape_names"]: array = module.manage_subscriptions(api, auth_count) subscription_array += array if jobs["scrape_paid_content"]: paid_contents = api.get_paid_content() paid_content = module.paid_content_scraper(api) apis.append(api) subscription_array = module.format_options( subscription_array, "usernames") stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') if exit_on_completion: print("Now exiting.") exit(0) elif not infinite_loop: print("Input anything to continue") input() elif loop_timeout: print('Pausing scraper for ' + loop_timeout + ' seconds.') time.sleep(int(loop_timeout)) except Exception as e: log_error.exception(e) input()
def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=True): json_settings = json_config["settings"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_scrape_names = json_site_settings["auto_scrape_names"] if isinstance(auto_scrape_names, str): temp_identifiers = auto_scrape_names.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] original_sessions = [] original_sessions = api_helper.create_session(settings=json_settings) original_sessions = [x for x in original_sessions if x] if not original_sessions: print("Unable to create session") return False session_manager = api_helper.session_manager() session_manager.sessions = original_sessions session_manager = api_helper.stimulate_sessions(session_manager) # offset_array = ["https://checkip.amazonaws.com"]*2 # pool = api_helper.multiprocessing() # count = 60 # while True: # test_ip2 = pool.starmap(multi, product( # offset_array, [original_sessions[0]]))[0] # if test_ip == test_ip2: # print(f"SAME IP: {test_ip2} - WAITING {count} second(s)") # time.sleep(count) # else: # print(f"NEW IP: {test_ip2} - TIME: {count}") # print # count+=1 # print archive_time = timeit.default_timer() if site_name_lower == "onlyfans": site_name = "OnlyFans" original_api = OnlyFans module = m_onlyfans if not apis: apis = main_helper.process_profiles(json_settings, session_manager, site_name, original_api) else: for api in apis: api.session_manager = session_manager subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] subscription_list = module.format_options(apis, "users") if not auto_profile_choice: print("Choose Profile") apis = choose_option(subscription_list, auto_profile_choice) apis = [x.pop(0) for x in apis] for api in apis: module.assign_vars(api.auth.auth_details, json_config, json_site_settings, site_name) setup = False setup, subscriptions = module.account_setup(api, identifiers, jobs) if not setup: auth_details = {} auth_details["auth"] = api.auth.auth_details.__dict__ profile_directory = api.auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( api.auth.profile_directory, "auth.json") main_helper.export_data(auth_details, user_auth_filepath) continue subscription_array += subscriptions subscription_list = module.format_options(subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis, identifiers) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names(module, subscription_list, auto_scrape_names, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) if webhooks: x = main_helper.process_webhooks(apis) elif site_name_lower == "starsavn": site_name = "StarsAVN" original_api = StarsAVN module = m_starsavn apis = main_helper.process_profiles(json_settings, original_sessions, site_name, original_api) auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] auth_count = -1 jobs = json_site_settings["jobs"] subscription_list = module.format_options(apis, "users") apis = choose_option(subscription_list, auto_profile_choice) apis = [x.pop(0) for x in apis] for api in apis: module.assign_vars(api.auth.auth_details, json_config, json_site_settings, site_name) identifier = "" setup = False setup = module.account_setup(api, identifier=identifier) if not setup: auth_details = api.auth.auth_details.__dict__ user_auth_filepath = os.path.join(api.auth.profile_directory, "auth.json") main_helper.export_data(auth_details, user_auth_filepath) continue if jobs["scrape_names"]: array = module.manage_subscriptions(api, auth_count, identifier=identifier) subscription_array += array subscription_list = module.format_options(subscription_array, "usernames") if jobs["scrape_paid_content"]: print("Scraping Paid Content") paid_content = module.paid_content_scraper(apis) if jobs["scrape_names"]: print("Scraping Subscriptions") names = main_helper.process_names(module, subscription_list, auto_scrape_names, apis, json_config, site_name_lower, site_name) x = main_helper.process_downloads(apis, module) stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print('Archive Completed in ' + stop_time + ' Minutes') session_manager.kill = True return apis
async def start_datascraper( json_config: dict[Any,Any], site_name_lower: str, api: Optional[OnlyFans.start| Fansly.start] = None, webhooks:bool=True, ) -> Optional[OnlyFans.start]: json_settings = json_config["settings"] json_webhooks = json_settings["webhooks"] json_sites = json_config["supported"] domain = json_settings["auto_site_choice"] main_helper.assign_vars(json_config) json_site_settings = json_sites[site_name_lower]["settings"] auto_model_choice = json_site_settings["auto_model_choice"] if isinstance(auto_model_choice, str): temp_identifiers = auto_model_choice.split(",") identifiers = [x for x in temp_identifiers if x] else: identifiers = [] auto_profile_choice = json_site_settings["auto_profile_choice"] subscription_array = [] proxies:list[str] = await api_helper.test_proxies(json_settings["proxies"]) if json_settings["proxies"] and not proxies: print("Unable to create session") return None archive_time = timeit.default_timer() match site_name_lower: case "onlyfans": site_name = "OnlyFans" module = m_onlyfans if not api: api = OnlyFans.start(max_threads=json_settings["max_threads"]) api.settings = json_config api = main_helper.process_profiles(json_settings, proxies, site_name, api) print subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options(api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option(subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars( auth.auth_details, json_config, json_site_settings, site_name ) setup = False setup, subscriptions = await module.account_setup( auth, identifiers, jobs, auth_count ) if not setup: if webhooks: await main_helper.process_webhooks(api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.export() profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json" ) main_helper.export_data(auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions await main_helper.process_webhooks(api, "auth_webhook", "succeeded") # Do stuff with authed user subscription_list = module.format_options( subscription_array, "usernames", api.auths ) if jobs["scrape_paid_content"] and api.has_active_auths(): print("Scraping Paid Content") await module.paid_content_scraper(api, identifiers) if jobs["scrape_names"] and api.has_active_auths(): print("Scraping Subscriptions") await main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name, ) await main_helper.process_downloads(api, module) if webhooks: await main_helper.process_webhooks(api, "download_webhook", "succeeded") case "fansly": site_name = "Fansly" module = m_fansly if not api: api = Fansly.start(max_threads=json_settings["max_threads"]) api.settings = json_config api = main_helper.process_profiles(json_settings, proxies, site_name, api) subscription_array = [] auth_count = 0 jobs = json_site_settings["jobs"] subscription_list = module.format_options(api.auths, "users") if not auto_profile_choice: print("Choose Profile") auths = choose_option(subscription_list, auto_profile_choice, True) api.auths = [x.pop(0) for x in auths] for auth in api.auths: if not auth.auth_details: continue module.assign_vars( auth.auth_details, json_config, json_site_settings, site_name ) setup = False setup, subscriptions = await module.account_setup( auth, identifiers, jobs, auth_count ) if not setup: if webhooks: await main_helper.process_webhooks(api, "auth_webhook", "failed") auth_details = {} auth_details["auth"] = auth.auth_details.export() profile_directory = auth.profile_directory if profile_directory: user_auth_filepath = os.path.join( auth.profile_directory, "auth.json" ) main_helper.export_data(auth_details, user_auth_filepath) continue auth_count += 1 subscription_array += subscriptions await main_helper.process_webhooks(api, "auth_webhook", "succeeded") # Do stuff with authed user subscription_list = module.format_options( subscription_array, "usernames", api.auths ) if jobs["scrape_paid_content"] and api.has_active_auths(): print("Scraping Paid Content") await module.paid_content_scraper(api, identifiers) if jobs["scrape_names"] and api.has_active_auths(): print("Scraping Subscriptions") await main_helper.process_names( module, subscription_list, auto_model_choice, api, json_config, site_name_lower, site_name, ) await main_helper.process_downloads(api, module) if webhooks: await main_helper.process_webhooks(api, "download_webhook", "succeeded") case "starsavn": pass # site_name = "StarsAVN" # original_api = StarsAVN # module = m_starsavn # apis = main_helper.process_profiles( # json_settings, original_sessions, site_name, original_api) # auto_profile_choice = json_site_settings["auto_profile_choice"] # subscription_array = [] # auth_count = -1 # jobs = json_site_settings["jobs"] # subscription_list = module.format_options( # apis, "users") # apis = choose_option( # subscription_list, auto_profile_choice) # apis = [x.pop(0) for x in apis] # for api in apis: # module.assign_vars(api.auth.auth_details, json_config, # json_site_settings, site_name) # identifier = "" # setup = False # setup = module.account_setup(api, identifier=identifier) # if not setup: # auth_details = api.auth.auth_details.__dict__ # user_auth_filepath = os.path.join( # api.auth.profile_directory, "auth.json") # main_helper.export_data( # auth_details, user_auth_filepath) # continue # if jobs["scrape_names"]: # array = module.manage_subscriptions( # api, auth_count, identifier=identifier) # subscription_array += array # subscription_list = module.format_options( # subscription_array, "usernames") # if jobs["scrape_paid_content"]: # print("Scraping Paid Content") # paid_content = module.paid_content_scraper(apis) # if jobs["scrape_names"]: # print("Scraping Subscriptions") # names = main_helper.process_names( # module, subscription_list, auto_model_choice, apis, json_config, site_name_lower, site_name) # x = main_helper.process_downloads(apis, module) stop_time = str(int(timeit.default_timer() - archive_time) / 60)[:4] print("Archive Completed in " + stop_time + " Minutes") return api