def get_web_session(cache_storage=None):
proxies = None
http_proxy = os.getenv('HTTP_PROXY')
https_proxy = os.getenv('HTTPS_PROXY')
if http_proxy is not None and https_proxy is not None:
proxies = {'http': http_proxy, 'https': https_proxy}
if cache_storage is not None:
o = urlparse(cache_storage)
if o.scheme == "mongodb":
# these requests-cache internals gymnastics are necessary
# because it will not create a database with the desired name otherwise
dbname = o.path.replace('/', '')
dbconn = MongoClient(cache_storage)
session = CachedSession(backend='mongodb')
session.cache = MongoCache(connection=dbconn, db_name=dbname)
else:
session = CachedSession(cache_name=cache_storage, extension='')
else:
session = Session()
retry = Retry(total=10,
read=10,
connect=10,
backoff_factor=0.3,
status_forcelist=(500, 502, 504))
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.proxies = proxies
session.headers.update({'User-Agent': __useragent__})
return session
