def start_datascraper():
    parser = ArgumentParser()
    parser.add_argument("-m",
                        "--metadata",
                        action='store_true',
                        help="only exports metadata")
    args = parser.parse_args()
    if args.metadata:
        print("Exporting Metadata Only")
    log_error = main_helper.setup_logger('errors', 'errors.log')
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s %(levelname)s %(name)s %(message)s')
    console.setFormatter(formatter)
    logging.getLogger("").addHandler(console)
    # root = os.getcwd()
    config_path = os.path.join('.settings', 'config.json')
    json_config, json_config2 = main_helper.get_config(config_path)
    json_settings = json_config["settings"]
    json_sites = json_config["supported"]
    infinite_loop = json_settings["infinite_loop"]
    global_user_agent = json_settings['global_user_agent']
    domain = json_settings["auto_site_choice"]
    path = os.path.join('.settings', 'extra_auth.json')
    # extra_auth_config, extra_auth_config2 = main_helper.get_config(path)
    extra_auth_config = {}
    exit_on_completion = json_settings['exit_on_completion']
    loop_timeout = json_settings['loop_timeout']
    main_helper.assign_vars(json_config)

    string, site_names = module_chooser(domain, json_sites)
    try:
        while True:
            if domain:
                if site_names:
                    site_name = domain
                else:
                    print(string)
                    continue
            else:
                print(string)
                x = input()
                if x == "x":
                    break
                x = int(x)
                site_name = site_names[x]
            site_name_lower = site_name.lower()

            json_auth_array = [json_sites[site_name_lower]["auth"]]

            json_site_settings = json_sites[site_name_lower]["settings"]
            auto_scrape_names = json_site_settings["auto_scrape_names"]
            extra_auth_settings = json_sites[site_name_lower][
                "extra_auth_settings"] if "extra_auth_settings" in json_sites[
                    site_name_lower] else {
                        "extra_auth": False
                    }
            extra_auth = extra_auth_settings["extra_auth"]
            if extra_auth:
                choose_auth = extra_auth_settings["choose_auth"]
                merge_auth = extra_auth_settings["merge_auth"]
                json_auth_array += extra_auth_config["supported"][
                    site_name_lower]["auths"]
                if choose_auth:
                    json_auth_array = main_helper.choose_auth(json_auth_array)
            apis = []
            module = m_onlyfans
            subscription_array = []
            legacy = True
            original_sessions = api_helper.create_session(
                settings=json_settings)
            if not original_sessions:
                print("Unable to create session")
                continue
            archive_time = timeit.default_timer()
            if site_name_lower == "onlyfans":
                site_name = "OnlyFans"
                subscription_array = []
                auth_count = -1
                jobs = json_site_settings["jobs"]
                for json_auth in json_auth_array:
                    api = OnlyFans.start(original_sessions)
                    auth_count += 1
                    user_agent = global_user_agent if not json_auth[
                        'user_agent'] else json_auth['user_agent']

                    module = m_onlyfans
                    module.assign_vars(json_auth, json_config,
                                       json_site_settings, site_name)
                    api.set_auth_details(**json_auth,
                                         global_user_agent=user_agent)
                    identifier = ""
                    setup = module.account_setup(api, identifier=identifier)
                    if not setup:
                        continue
                    if jobs["scrape_names"]:
                        array = module.manage_subscriptions(
                            api, auth_count, identifier=identifier)
                        subscription_array += array
                    apis.append(api)
                subscription_list = module.format_options(
                    subscription_array, "usernames")
                if jobs["scrape_paid_content"]:
                    print("Scraping Paid Content")
                    paid_content = module.paid_content_scraper(apis)
                if jobs["scrape_names"]:
                    print("Scraping Subscriptions")
                    x = main_helper.process_names(module, subscription_list,
                                                  auto_scrape_names,
                                                  json_auth_array, apis,
                                                  json_config, site_name_lower,
                                                  site_name)
                x = main_helper.process_downloads(apis, module)
                print
            elif site_name_lower == "starsavn":
                site_name = "StarsAVN"
                subscription_array = []
                auth_count = -1
                for json_auth in json_auth_array:
                    sessions = api_helper.copy_sessions(original_sessions)
                    api = StarsAVN.start(sessions)
                    auth_count += 1
                    user_agent = global_user_agent if not json_auth[
                        'user_agent'] else json_auth['user_agent']

                    module = m_starsavn
                    module.assign_vars(json_auth, json_config,
                                       json_site_settings, site_name)
                    api.set_auth_details(**json_auth,
                                         global_user_agent=user_agent)
                    setup = module.account_setup(api)
                    if not setup:
                        continue
                    jobs = json_site_settings["jobs"]
                    if jobs["scrape_names"]:
                        array = module.manage_subscriptions(api, auth_count)
                        subscription_array += array
                    if jobs["scrape_paid_content"]:
                        paid_contents = api.get_paid_content()
                        paid_content = module.paid_content_scraper(api)
                    apis.append(api)
                subscription_array = module.format_options(
                    subscription_array, "usernames")
            stop_time = str(int(timeit.default_timer() - archive_time) /
                            60)[:4]
            print('Archive Completed in ' + stop_time + ' Minutes')
            if exit_on_completion:
                print("Now exiting.")
                exit(0)
            elif not infinite_loop:
                print("Input anything to continue")
                input()
            elif loop_timeout:
                print('Pausing scraper for ' + loop_timeout + ' seconds.')
                time.sleep(int(loop_timeout))
    except Exception as e:
        log_error.exception(e)
        input()
Example #2
0
main_test.check_config()
main_test.check_profiles()

if __name__ == "__main__":
    import datascraper.main_datascraper as main_datascraper
    import helpers.main_helper as main_helper

    config_path = os.path.join(".settings", "config.json")
    json_config, json_config2 = main_helper.get_config(config_path)
    json_settings = json_config["settings"]
    exit_on_completion = json_settings["exit_on_completion"]
    infinite_loop = json_settings["infinite_loop"]
    loop_timeout = json_settings["loop_timeout"]
    json_sites = json_config["supported"]
    domain = json_settings["auto_site_choice"]
    string, site_names = main_helper.module_chooser(domain, json_sites)

    # logging.basicConfig(level=logging.DEBUG, format="%(message)s")
    while True:
        try:
            if domain:
                if site_names:
                    site_name = domain
                else:
                    print(string)
                    continue
            else:
                print(string)
                x = input()
                if x == "x":
                    break
Example #3
0
if __name__ == "__main__":
    import apis.api_helper as api_helper
    import datascraper.main_datascraper as main_datascraper
    import helpers.main_helper as main_helper

    api_helper.parsed_args = parsed_args

    config_path = Path(".settings", "config.json")
    config, _updated = main_helper.get_config(config_path)
    global_settings = config.settings
    exit_on_completion = global_settings.exit_on_completion
    infinite_loop = global_settings.infinite_loop
    loop_timeout = global_settings.loop_timeout
    domain = global_settings.auto_site_choice
    json_sites = config.supported
    string, site_names_ = main_helper.module_chooser(domain,
                                                     json_sites.__dict__)
    site_name_literals = Literal["OnlyFans", "Fansly", "StarsAVN"]
    site_names: list[site_name_literals] = list(get_args(site_name_literals))

    # logging.basicConfig(level=logging.DEBUG, format="%(message)s")
    async def main():
        while True:
            site_options = OptionsFormat(site_names, "sites", domain)
            for site_name in site_options.final_choices:
                api = await main_datascraper.start_datascraper(
                    config, site_name)
                if api:
                    api.close_pools()
                    await asyncio.sleep(1)
            if exit_on_completion:
                print("Now exiting.")