def download_task(url, **kwargs):
"""a simple task to use requests to get a url. By default, we return
the raw response.
Parameters
==========
REQUIRED:
url: a url to download (stream)
OPTIONAL:
write_format: to change from default "w"
disable_ssl_check: set to anything to not verify (not recommended)
"""
result = None
# Update the user what we are doing
bot.verbose("Downloading %s" % url)
# Use the basename or the user set file_name to write to
file_name = kwargs.get("file_name", os.path.basename(url))
destination = os.path.join(tempfile.gettempdir(), file_name)
verify = True
# Does the user want to disable ssl?
if "disable_ssl_check" in kwargs:
if kwargs["disable_ssl_check"]:
bot.warning(
"Verify of certificates disabled! ::TESTING USE ONLY::")
verify = False
# If the user doesn't want to write, but maybe write binary
fmt = kwargs.get("write_format", "wb")
headers = get_headers(kwargs)
# Does the url being requested exist?
if requests.head(url, verify=verify,
headers=headers).status_code in [200, 401]:
# Stream the response
response = requests.get(url,
verify=verify,
stream=True,
headers=headers)
# Invalid permissions
if response.status_code == 401:
return result
# Successful, stream to result destination
if response.status_code == 200:
chunk_size = 1 << 20
with open(destination, fmt) as filey:
for chunk in response.iter_content(chunk_size=chunk_size):
filey.write(chunk)
result = destination
return result
def getenv(variable_key, default=None, required=False, silent=True):
"""attempt to get an environment variable. If the variable
is not found, None is returned.
Parameters
==========
variable_key: the variable name
required: exit with error if not found
silent: Do not print debugging information for variable
"""
variable = os.environ.get(variable_key, default)
if variable is None and required:
bot.error("Cannot find environment variable %s, exiting." %
variable_key)
sys.exit(1)
if not silent and variable is not None:
bot.verbose("%s found as %s" % (variable_key, variable))
return variable