def __config_headers():
"""
configure the request headers, this will configure user agents and proxies
"""
if opt.proxyConfig is not None:
proxy = opt.proxyConfig
else:
proxy = None
if opt.usePersonalAgent is not None:
agent = opt.usePersonalAgent
elif opt.useRandomAgent:
agent = grab_random_agent(verbose=opt.runInVerbose)
else:
agent = None
return proxy, agent
def __config_headers():
"""
configure the request headers, this will configure user agents and proxies
"""
if opt.proxyConfig is not None:
proxy = opt.proxyConfig
elif opt.proxyFileRand is not None:
if opt.runInVerbose:
logger.debug(set_color(
"loading random proxy from '{}'...".format(opt.proxyFileRand), level=10
))
with open(opt.proxyFileRand) as proxies:
possible = proxies.readlines()
proxy = random.choice(possible).strip()
else:
proxy = None
if opt.usePersonalAgent is not None:
agent = opt.usePersonalAgent
elif opt.useRandomAgent:
agent = grab_random_agent(verbose=opt.runInVerbose)
else:
agent = None
return proxy, agent