def download(url, file_name, headers=None, show_progress=True):
"""stream to a temporary file, rename on successful completion
Parameters
==========
file_name: the file name to stream to
url: the url to stream from
headers: additional headers to add
"""
fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name))
os.close(fd)
if DISABLE_SSL_CHECK is True:
bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::")
verify = not DISABLE_SSL_CHECK
# Does the url being requested exist?
if requests.head(url, verify=verify).status_code in [200, 401]:
response = stream(url, headers=headers, stream_to=tmp_file)
if isinstance(response, HTTPError):
bot.error("Error downloading %s, exiting." % url)
sys.exit(1)
shutil.move(tmp_file, file_name)
else:
bot.error("Invalid url or permissions %s" % url)
return file_name
def download_task(url, headers, destination, download_type="layer"):
"""download an image layer (.tar.gz) to a specified download folder.
This task is done by using local versions of the same download functions
that are used for the client.
core stream/download functions of the parent client.
Parameters
==========
image_id: the shasum id of the layer, already determined to not exist
repo_name: the image name (library/ubuntu) to retrieve
download_folder: download to this folder. If not set, uses temp.
"""
# Update the user what we are doing
bot.verbose("Downloading %s from %s" % (download_type, url))
# Step 1: Download the layer atomically
file_name = "%s.%s" % (destination, next(tempfile._get_candidate_names()))
tar_download = download(url, file_name, headers=headers)
try:
shutil.move(tar_download, destination)
except Exception:
msg = "Cannot untar layer %s," % tar_download
msg += " was there a problem with download?"
bot.error(msg)
sys.exit(1)
return destination
def download(self, url, file_name, headers=None, show_progress=True):
"""stream to a temporary file, rename on successful completion
Parameters
==========
file_name: the file name to stream to
url: the url to stream from
headers: additional headers to add
force: If the final image exists, don't overwrite
"""
fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name))
os.close(fd)
# Should we verify the request?
verify = self._verify()
# Check here if exists
if requests.head(url, verify=verify).status_code in [200, 401]:
response = self.stream(url, headers=headers, stream_to=tmp_file)
if isinstance(response, HTTPError):
bot.error("Error downloading %s, exiting." % url)
sys.exit(1)
shutil.move(tmp_file, file_name)
else:
bot.error("Invalid url or permissions %s" % url)
return file_name
def mkdir_p(path):
"""mkdir_p attempts to get the same functionality as mkdir -p
:param path: the path to create.
"""
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def check_env(self, envar, value):
"""ensure that variable envar is set to some value,
otherwise exit on error.
Parameters
==========
envar: the environment variable name
value: the setting that shouldn't be None
"""
if value is None:
bot.error("You must export %s to use Github" % envar)
print("https://vsoch.github.io/helpme/helper-github")
sys.exit(1)
def stream(url, headers, stream_to=None, retry=True):
"""stream is a get that will stream to file_name. Since this is a worker
task, it differs from the client provided version in that it requires
headers.
"""
bot.debug("GET %s" % url)
if DISABLE_SSL_CHECK is True:
bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::")
# Ensure headers are present, update if not
response = requests.get(
url, headers=headers, verify=not DISABLE_SSL_CHECK, stream=True
)
# Deal with token if necessary
if response.status_code == 401 and retry is True:
headers = update_token(response, headers)
return stream(url, headers, stream_to, retry=False)
if response.status_code == 200:
# Keep user updated with Progress Bar
content_size = None
if "Content-Length" in response.headers:
progress = 0
content_size = int(response.headers["Content-Length"])
bot.show_progress(progress, content_size, length=35)
chunk_size = 1 << 20
with open(stream_to, "wb") as filey:
for chunk in response.iter_content(chunk_size=chunk_size):
filey.write(chunk)
if content_size is not None:
progress += chunk_size
bot.show_progress(
iteration=progress,
total=content_size,
length=35,
carriage_return=False,
)
# Newline to finish download
sys.stdout.write("\n")
return stream_to
bot.error("Problem with stream, response %s" % (response.status_code))
sys.exit(1)
def healthy(self, url):
"""determine if a resource is healthy based on an accepted response (200)
or redirect (301)
Parameters
==========
url: the URL to check status for, based on the status_code of HEAD
"""
response = requests.get(url)
status_code = response.status_code
if status_code != 200:
bot.error("%s, response status code %s." % (url, status_code))
return False
return True
def upload_asciinema(filename):
"""a wrapper around generation of an asciinema.api.Api to call the
upload command given an already existing asciinema file.
Parameters
==========
filename: the asciinema file to upload, can be generated with
function record_asciinema in record.py
"""
if os.path.exists(filename):
try:
from asciinema.commands.upload import UploadCommand
import asciinema.config as aconfig
from asciinema.api import Api
except:
bot.exit(
"The asciinema module is required to submit "
"an asciinema recording. Try pip install helpme[asciinema]"
)
# Load the API class
cfg = aconfig.load()
api = Api(cfg.api_url, os.environ.get("USER"), cfg.install_id)
# Perform the upload, return the url
uploader = UploadCommand(api, filename)
try:
url, warn = uploader.api.upload_asciicast(filename)
if warn:
uploader.print_warning(warn)
# Extract just the url, if provided (always is https)
if url:
match = re.search("https://.+", url)
if match:
url = match.group()
return url
except:
bot.error("Problem with upload, skipping")
else:
bot.warning("Cannot find %s, skipping submission." % filename)
def load_secrets(self):
required_vars = ["subdomain", "api_key", "api_secret", "email"]
for required in required_vars:
envar = "HELPME_USERVOICE_%s" % required.upper()
# The settings can be provided in either config, depends on install
user_setting = self._get_and_update_setting(envar, user=True)
setting = self._get_and_update_setting(envar, user=False)
setting = setting or user_setting
if not setting:
bot.error("export %s environment or add to helpme.cfg" % envar)
sys.exit(1)
setattr(self, required, setting)
def stream(self,
url,
headers=None,
stream_to=None,
retry=True,
default_headers=True):
"""
stream is a get that will stream to file_name. This stream is intended
to take a url and (optionally) a set of headers and file to stream to,
and will generate a response with requests.get.
Parameters
==========
url: the url to do a requests.get to
headers: any updated headers to use for the requets
stream_to: the file to stream to
retry: should the client retry? (intended for use after token refresh)
by default we retry once after token refresh, then fail.
"""
bot.debug("GET %s" % url)
# Ensure headers are present, update if not
if headers == None:
if self.headers is None:
self._reset_headers()
headers = self.headers.copy()
response = requests.get(url,
headers=headers,
verify=self._verify(),
stream=True)
# Deal with token if necessary
if response.status_code == 401 and retry is True:
if hasattr(self, "_update_token"):
self._update_token(response)
return self.stream(url, headers, stream_to, retry=False)
if response.status_code == 200:
return self._stream(response, stream_to=stream_to)
bot.error("Problem with stream, response %s" % (response.status_code))
sys.exit(1)
def stream_response(self, response, stream_to=None):
"""
stream response is one level higher up than stream, starting with a
response object and then performing the stream without making the
requests.get. The expectation is that the request was successful
(status code 20*).
Parameters
==========
response: a response that is ready to be iterated over to download in
streamed chunks
stream_to: the file to stream to
"""
if response.status_code == 200:
# Keep user updated with Progress Bar
content_size = None
if "Content-Length" in response.headers:
progress = 0
content_size = int(response.headers["Content-Length"])
bot.show_progress(progress, content_size, length=35)
chunk_size = 1 << 20
with open(stream_to, "wb") as filey:
for chunk in response.iter_content(chunk_size=chunk_size):
filey.write(chunk)
if content_size is not None:
progress += chunk_size
bot.show_progress(
iteration=progress,
total=content_size,
length=35,
carriage_return=False,
)
# Newline to finish download
sys.stdout.write("\n")
return stream_to
bot.error("Problem with stream, response %s" % (response.status_code))
sys.exit(1)
def create_post(self, title, body, board, category, username):
"""create a Discourse post, given a title, body, board, and token.
Parameters
==========
title: the issue title
body: the issue body
board: the discourse board to post to
"""
category_url = "%s/categories.json" % board
response = requests.get(category_url)
if response.status_code != 200:
print("Error with retrieving %s" % category_url)
sys.exit(1)
# Get a list of all categories
categories = response.json()["category_list"]["categories"]
categories = {c["name"]: c["id"] for c in categories}
# And if not valid, warn the user
if category not in categories:
bot.warning("%s is not valid, will use default" % category)
category_id = categories.get(category, None)
headers = {
"Content-Type": "application/json",
"User-Api-Client-Id": self.client_id,
"User-Api-Key": self.token,
}
# First get the category ids
data = {"title": title, "raw": body, "category": category_id}
response = requests.post("%s/posts.json" % board,
headers=headers,
data=json.dumps(data))
if response.status_code in [200, 201, 202]:
topic = response.json()
url = "%s/t/%s/%s" % (board, topic["topic_slug"], topic["topic_id"])
bot.info(url)
return url
elif response.status_code == 404:
bot.error("Cannot post to board, not found. Do you have permission?")
sys.exit(1)
else:
bot.error("Cannot post to board %s" % board)
bot.error(response.content)
sys.exit(1)
def update_token(response, headers):
"""update_token uses HTTP basic authentication to attempt to authenticate
given a 401 response. We take as input previous headers, and update
them.
Parameters
==========
response: the http request response to parse for the challenge.
"""
not_asking_auth = "Www-Authenticate" not in response.headers
if response.status_code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?'
match = re.match(regexp, challenge)
if not match:
bot.error("Unrecognized authentication challenge, exiting.")
sys.exit(1)
realm = match.group(1)
service = match.group(2)
scope = match.group(3).split(",")[0]
token_url = realm + "?service=" + service + "&expires_in=900&scope=" + scope
response = get(token_url)
try:
token = response["token"]
token = {"Authorization": "Bearer %s" % token}
headers.update(token)
except Exception:
bot.error("Error getting token.")
sys.exit(1)
return headers
def create_issue(title, body, repo, token):
"""create a Github issue, given a title, body, repo, and token.
Parameters
==========
title: the issue title
body: the issue body
repo: the full name of the repo
token: the user's personal Github token
"""
owner, name = repo.split("/")
url = "https://api.github.com/repos/%s/%s/issues" % (owner, name)
data = {"title": title, "body": body}
headers = {
"Authorization": "token %s" % token,
"Accept": "application/vnd.github.symmetra-preview+json",
}
response = requests.post(url, data=json.dumps(data), headers=headers)
if response.status_code in [201, 202]:
url = response.json()["html_url"]
bot.info(url)
return url
elif response.status_code == 404:
bot.error("Cannot create issue. Does your token have scope repo?")
sys.exit(1)
else:
bot.error("Cannot create issue %s" % title)
bot.error(response.content)
sys.exit(1)
def call(
self,
url,
func,
data=None,
headers=None,
return_json=True,
stream=False,
retry=True,
default_headers=True,
quiet=False,
):
"""call will issue the call, and issue a refresh token
given a 401 response, and if the client has a _update_token function
Parameters
==========
func: the function (eg, post, get) to call
url: the url to send file to
headers: if not None, update the client self.headers with dictionary
data: additional data to add to the request
return_json: return json if successful
default_headers: use the client's self.headers (default True)
"""
if data is not None:
if not isinstance(data, dict):
data = json.dumps(data)
heads = dict()
if default_headers is True:
heads = self.headers.copy()
if headers is not None:
if isinstance(headers, dict):
heads.update(headers)
response = func(url=url,
headers=heads,
data=data,
verify=self._verify(),
stream=stream)
# Errored response, try again with refresh
if response.status_code in [500, 502]:
bot.error("Beep boop! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 404:
# Not found, we might want to continue on
if quiet is False:
bot.error("Beep boop! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 401:
# If client has method to update token, try it once
if retry is True and hasattr(self, "_update_token"):
# A result of None indicates no update to the call
self._update_token(response)
return self._call(
url,
func,
data=data,
headers=headers,
return_json=return_json,
stream=stream,
retry=False,
)
bot.error("Your credentials are expired! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
elif response.status_code == 200:
if return_json:
try:
response = response.json()
except ValueError:
bot.error("The server returned a malformed response.")
sys.exit(1)
return response
def run(self, func, tasks, func2=None):
"""run will send a list of tasks,
a tuple with arguments, through a function.
the arguments should be ordered correctly.
:param func: the function to run with multiprocessing.pool
:param tasks: a list of tasks, each a tuple
of arguments to process
:param func2: filter function to run result
from func through (optional)
"""
# Keep track of some progress for the user
progress = 1
total = len(tasks)
# if we don't have tasks, don't run
if len(tasks) == 0:
return
# If two functions are run per task, double total jobs
if func2 is not None:
total = total * 2
finished = []
level1 = []
results = []
try:
prefix = "[%s/%s]" % (progress, total)
bot.show_progress(0, total, length=35, prefix=prefix)
pool = multiprocessing.Pool(self.workers, init_worker)
self.start()
for task in tasks:
result = pool.apply_async(multi_wrapper,
multi_package(func, [task]))
results.append(result)
level1.append(result._job)
while len(results) > 0:
result = results.pop()
result.wait()
bot.show_progress(progress, total, length=35, prefix=prefix)
progress += 1
prefix = "[%s/%s]" % (progress, total)
# Pass the result through a second function?
if func2 is not None and result._job in level1:
result = pool.apply_async(
multi_wrapper, multi_package(func2,
[(result.get(), )]))
results.append(result)
else:
finished.append(result.get())
self.end()
pool.close()
pool.join()
except (KeyboardInterrupt, SystemExit):
bot.error("Keyboard interrupt detected, terminating workers!")
pool.terminate()
sys.exit(1)
except Exception as e:
bot.error(e)
return finished
def _submit(self):
"""if this function is called, it indicates the helper submodule
doesn't have a submit function, so no submission is done.
"""
bot.error("_submit() not implemented in helper %s." % self.name)
sys.exit(1)
def call(
url, func, data=None, headers=None, return_json=True, stream=False, retry=True
):
"""call will issue the call, and issue a refresh token
given a 401 response, and if the client has a _update_token function
Parameters
==========
func: the function (eg, post, get) to call
url: the url to send file to
headers: headers for the request
data: additional data to add to the request
return_json: return json if successful
"""
if DISABLE_SSL_CHECK is True:
bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::")
if data is not None:
if not isinstance(data, dict):
data = json.dumps(data)
response = func(
url=url, headers=headers, data=data, verify=not DISABLE_SSL_CHECK, stream=stream
)
# Errored response, try again with refresh
if response.status_code in [500, 502]:
bot.error("Beep boop! %s: %s" % (response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 404:
bot.error("Beep boop! %s: %s" % (response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 401:
# If client has method to update token, try it once
if retry is True:
# A result of None indicates no update to the call
headers = update_token(response, headers)
return call(
url,
func,
data=data,
headers=headers,
return_json=return_json,
stream=stream,
retry=False,
)
bot.error(
"Your credentials are expired! %s: %s"
% (response.reason, response.status_code)
)
sys.exit(1)
elif response.status_code == 200:
if return_json:
try:
response = response.json()
except ValueError:
bot.error("The server returned a malformed response.")
sys.exit(1)
return response
