def download(url, file_name, headers=None, show_progress=True): """stream to a temporary file, rename on successful completion Parameters ========== file_name: the file name to stream to url: the url to stream from headers: additional headers to add """ fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name)) os.close(fd) if DISABLE_SSL_CHECK is True: bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::") verify = not DISABLE_SSL_CHECK # Does the url being requested exist? if requests.head(url, verify=verify).status_code in [200, 401]: response = stream(url, headers=headers, stream_to=tmp_file) if isinstance(response, HTTPError): bot.error("Error downloading %s, exiting." % url) sys.exit(1) shutil.move(tmp_file, file_name) else: bot.error("Invalid url or permissions %s" % url) return file_name
def download_task(url, headers, destination, download_type="layer"): """download an image layer (.tar.gz) to a specified download folder. This task is done by using local versions of the same download functions that are used for the client. core stream/download functions of the parent client. Parameters ========== image_id: the shasum id of the layer, already determined to not exist repo_name: the image name (library/ubuntu) to retrieve download_folder: download to this folder. If not set, uses temp. """ # Update the user what we are doing bot.verbose("Downloading %s from %s" % (download_type, url)) # Step 1: Download the layer atomically file_name = "%s.%s" % (destination, next(tempfile._get_candidate_names())) tar_download = download(url, file_name, headers=headers) try: shutil.move(tar_download, destination) except Exception: msg = "Cannot untar layer %s," % tar_download msg += " was there a problem with download?" bot.error(msg) sys.exit(1) return destination
def download(self, url, file_name, headers=None, show_progress=True): """stream to a temporary file, rename on successful completion Parameters ========== file_name: the file name to stream to url: the url to stream from headers: additional headers to add force: If the final image exists, don't overwrite """ fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name)) os.close(fd) # Should we verify the request? verify = self._verify() # Check here if exists if requests.head(url, verify=verify).status_code in [200, 401]: response = self.stream(url, headers=headers, stream_to=tmp_file) if isinstance(response, HTTPError): bot.error("Error downloading %s, exiting." % url) sys.exit(1) shutil.move(tmp_file, file_name) else: bot.error("Invalid url or permissions %s" % url) return file_name
def mkdir_p(path): """mkdir_p attempts to get the same functionality as mkdir -p :param path: the path to create. """ try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(path): pass else: bot.error("Error creating path %s, exiting." % path) sys.exit(1)
def check_env(self, envar, value): """ensure that variable envar is set to some value, otherwise exit on error. Parameters ========== envar: the environment variable name value: the setting that shouldn't be None """ if value is None: bot.error("You must export %s to use Github" % envar) print("https://vsoch.github.io/helpme/helper-github") sys.exit(1)
def stream(url, headers, stream_to=None, retry=True): """stream is a get that will stream to file_name. Since this is a worker task, it differs from the client provided version in that it requires headers. """ bot.debug("GET %s" % url) if DISABLE_SSL_CHECK is True: bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::") # Ensure headers are present, update if not response = requests.get( url, headers=headers, verify=not DISABLE_SSL_CHECK, stream=True ) # Deal with token if necessary if response.status_code == 401 and retry is True: headers = update_token(response, headers) return stream(url, headers, stream_to, retry=False) if response.status_code == 200: # Keep user updated with Progress Bar content_size = None if "Content-Length" in response.headers: progress = 0 content_size = int(response.headers["Content-Length"]) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, "wb") as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress( iteration=progress, total=content_size, length=35, carriage_return=False, ) # Newline to finish download sys.stdout.write("\n") return stream_to bot.error("Problem with stream, response %s" % (response.status_code)) sys.exit(1)
def healthy(self, url): """determine if a resource is healthy based on an accepted response (200) or redirect (301) Parameters ========== url: the URL to check status for, based on the status_code of HEAD """ response = requests.get(url) status_code = response.status_code if status_code != 200: bot.error("%s, response status code %s." % (url, status_code)) return False return True
def upload_asciinema(filename): """a wrapper around generation of an asciinema.api.Api to call the upload command given an already existing asciinema file. Parameters ========== filename: the asciinema file to upload, can be generated with function record_asciinema in record.py """ if os.path.exists(filename): try: from asciinema.commands.upload import UploadCommand import asciinema.config as aconfig from asciinema.api import Api except: bot.exit( "The asciinema module is required to submit " "an asciinema recording. Try pip install helpme[asciinema]" ) # Load the API class cfg = aconfig.load() api = Api(cfg.api_url, os.environ.get("USER"), cfg.install_id) # Perform the upload, return the url uploader = UploadCommand(api, filename) try: url, warn = uploader.api.upload_asciicast(filename) if warn: uploader.print_warning(warn) # Extract just the url, if provided (always is https) if url: match = re.search("https://.+", url) if match: url = match.group() return url except: bot.error("Problem with upload, skipping") else: bot.warning("Cannot find %s, skipping submission." % filename)
def load_secrets(self): required_vars = ["subdomain", "api_key", "api_secret", "email"] for required in required_vars: envar = "HELPME_USERVOICE_%s" % required.upper() # The settings can be provided in either config, depends on install user_setting = self._get_and_update_setting(envar, user=True) setting = self._get_and_update_setting(envar, user=False) setting = setting or user_setting if not setting: bot.error("export %s environment or add to helpme.cfg" % envar) sys.exit(1) setattr(self, required, setting)
def stream(self, url, headers=None, stream_to=None, retry=True, default_headers=True): """ stream is a get that will stream to file_name. This stream is intended to take a url and (optionally) a set of headers and file to stream to, and will generate a response with requests.get. Parameters ========== url: the url to do a requests.get to headers: any updated headers to use for the requets stream_to: the file to stream to retry: should the client retry? (intended for use after token refresh) by default we retry once after token refresh, then fail. """ bot.debug("GET %s" % url) # Ensure headers are present, update if not if headers == None: if self.headers is None: self._reset_headers() headers = self.headers.copy() response = requests.get(url, headers=headers, verify=self._verify(), stream=True) # Deal with token if necessary if response.status_code == 401 and retry is True: if hasattr(self, "_update_token"): self._update_token(response) return self.stream(url, headers, stream_to, retry=False) if response.status_code == 200: return self._stream(response, stream_to=stream_to) bot.error("Problem with stream, response %s" % (response.status_code)) sys.exit(1)
def stream_response(self, response, stream_to=None): """ stream response is one level higher up than stream, starting with a response object and then performing the stream without making the requests.get. The expectation is that the request was successful (status code 20*). Parameters ========== response: a response that is ready to be iterated over to download in streamed chunks stream_to: the file to stream to """ if response.status_code == 200: # Keep user updated with Progress Bar content_size = None if "Content-Length" in response.headers: progress = 0 content_size = int(response.headers["Content-Length"]) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, "wb") as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress( iteration=progress, total=content_size, length=35, carriage_return=False, ) # Newline to finish download sys.stdout.write("\n") return stream_to bot.error("Problem with stream, response %s" % (response.status_code)) sys.exit(1)
def create_post(self, title, body, board, category, username): """create a Discourse post, given a title, body, board, and token. Parameters ========== title: the issue title body: the issue body board: the discourse board to post to """ category_url = "%s/categories.json" % board response = requests.get(category_url) if response.status_code != 200: print("Error with retrieving %s" % category_url) sys.exit(1) # Get a list of all categories categories = response.json()["category_list"]["categories"] categories = {c["name"]: c["id"] for c in categories} # And if not valid, warn the user if category not in categories: bot.warning("%s is not valid, will use default" % category) category_id = categories.get(category, None) headers = { "Content-Type": "application/json", "User-Api-Client-Id": self.client_id, "User-Api-Key": self.token, } # First get the category ids data = {"title": title, "raw": body, "category": category_id} response = requests.post("%s/posts.json" % board, headers=headers, data=json.dumps(data)) if response.status_code in [200, 201, 202]: topic = response.json() url = "%s/t/%s/%s" % (board, topic["topic_slug"], topic["topic_id"]) bot.info(url) return url elif response.status_code == 404: bot.error("Cannot post to board, not found. Do you have permission?") sys.exit(1) else: bot.error("Cannot post to board %s" % board) bot.error(response.content) sys.exit(1)
def update_token(response, headers): """update_token uses HTTP basic authentication to attempt to authenticate given a 401 response. We take as input previous headers, and update them. Parameters ========== response: the http request response to parse for the challenge. """ not_asking_auth = "Www-Authenticate" not in response.headers if response.status_code != 401 or not_asking_auth: bot.error("Authentication error, exiting.") sys.exit(1) challenge = response.headers["Www-Authenticate"] regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?' match = re.match(regexp, challenge) if not match: bot.error("Unrecognized authentication challenge, exiting.") sys.exit(1) realm = match.group(1) service = match.group(2) scope = match.group(3).split(",")[0] token_url = realm + "?service=" + service + "&expires_in=900&scope=" + scope response = get(token_url) try: token = response["token"] token = {"Authorization": "Bearer %s" % token} headers.update(token) except Exception: bot.error("Error getting token.") sys.exit(1) return headers
def create_issue(title, body, repo, token): """create a Github issue, given a title, body, repo, and token. Parameters ========== title: the issue title body: the issue body repo: the full name of the repo token: the user's personal Github token """ owner, name = repo.split("/") url = "https://api.github.com/repos/%s/%s/issues" % (owner, name) data = {"title": title, "body": body} headers = { "Authorization": "token %s" % token, "Accept": "application/vnd.github.symmetra-preview+json", } response = requests.post(url, data=json.dumps(data), headers=headers) if response.status_code in [201, 202]: url = response.json()["html_url"] bot.info(url) return url elif response.status_code == 404: bot.error("Cannot create issue. Does your token have scope repo?") sys.exit(1) else: bot.error("Cannot create issue %s" % title) bot.error(response.content) sys.exit(1)
def call( self, url, func, data=None, headers=None, return_json=True, stream=False, retry=True, default_headers=True, quiet=False, ): """call will issue the call, and issue a refresh token given a 401 response, and if the client has a _update_token function Parameters ========== func: the function (eg, post, get) to call url: the url to send file to headers: if not None, update the client self.headers with dictionary data: additional data to add to the request return_json: return json if successful default_headers: use the client's self.headers (default True) """ if data is not None: if not isinstance(data, dict): data = json.dumps(data) heads = dict() if default_headers is True: heads = self.headers.copy() if headers is not None: if isinstance(headers, dict): heads.update(headers) response = func(url=url, headers=heads, data=data, verify=self._verify(), stream=stream) # Errored response, try again with refresh if response.status_code in [500, 502]: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 404: # Not found, we might want to continue on if quiet is False: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 401: # If client has method to update token, try it once if retry is True and hasattr(self, "_update_token"): # A result of None indicates no update to the call self._update_token(response) return self._call( url, func, data=data, headers=headers, return_json=return_json, stream=stream, retry=False, ) bot.error("Your credentials are expired! %s: %s" % (response.reason, response.status_code)) sys.exit(1) elif response.status_code == 200: if return_json: try: response = response.json() except ValueError: bot.error("The server returned a malformed response.") sys.exit(1) return response
def run(self, func, tasks, func2=None): """run will send a list of tasks, a tuple with arguments, through a function. the arguments should be ordered correctly. :param func: the function to run with multiprocessing.pool :param tasks: a list of tasks, each a tuple of arguments to process :param func2: filter function to run result from func through (optional) """ # Keep track of some progress for the user progress = 1 total = len(tasks) # if we don't have tasks, don't run if len(tasks) == 0: return # If two functions are run per task, double total jobs if func2 is not None: total = total * 2 finished = [] level1 = [] results = [] try: prefix = "[%s/%s]" % (progress, total) bot.show_progress(0, total, length=35, prefix=prefix) pool = multiprocessing.Pool(self.workers, init_worker) self.start() for task in tasks: result = pool.apply_async(multi_wrapper, multi_package(func, [task])) results.append(result) level1.append(result._job) while len(results) > 0: result = results.pop() result.wait() bot.show_progress(progress, total, length=35, prefix=prefix) progress += 1 prefix = "[%s/%s]" % (progress, total) # Pass the result through a second function? if func2 is not None and result._job in level1: result = pool.apply_async( multi_wrapper, multi_package(func2, [(result.get(), )])) results.append(result) else: finished.append(result.get()) self.end() pool.close() pool.join() except (KeyboardInterrupt, SystemExit): bot.error("Keyboard interrupt detected, terminating workers!") pool.terminate() sys.exit(1) except Exception as e: bot.error(e) return finished
def _submit(self): """if this function is called, it indicates the helper submodule doesn't have a submit function, so no submission is done. """ bot.error("_submit() not implemented in helper %s." % self.name) sys.exit(1)
def call( url, func, data=None, headers=None, return_json=True, stream=False, retry=True ): """call will issue the call, and issue a refresh token given a 401 response, and if the client has a _update_token function Parameters ========== func: the function (eg, post, get) to call url: the url to send file to headers: headers for the request data: additional data to add to the request return_json: return json if successful """ if DISABLE_SSL_CHECK is True: bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::") if data is not None: if not isinstance(data, dict): data = json.dumps(data) response = func( url=url, headers=headers, data=data, verify=not DISABLE_SSL_CHECK, stream=stream ) # Errored response, try again with refresh if response.status_code in [500, 502]: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 404: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 401: # If client has method to update token, try it once if retry is True: # A result of None indicates no update to the call headers = update_token(response, headers) return call( url, func, data=data, headers=headers, return_json=return_json, stream=stream, retry=False, ) bot.error( "Your credentials are expired! %s: %s" % (response.reason, response.status_code) ) sys.exit(1) elif response.status_code == 200: if return_json: try: response = response.json() except ValueError: bot.error("The server returned a malformed response.") sys.exit(1) return response