def __init__(self, location=PATH["CACHE"], name="config"): config = getenv("CONFIG") if not config: self.name = "%s.ini" % name self.location = location makedirs(self.location, exist_ok=True) else: self.name = osp.basename(config) self.location = osp.dirname(config) self.config = self.read()
def get_parser(): parser = argparse.ArgumentParser( prog = __command__, description = _DESCRIPTION_JUMBOTRON, add_help = False, formatter_class = ArgumentParserFormatter ) parser.add_argument("--pip-path", action = "append", help = "Path to pip executable to be used." ) parser.add_argument("-y", "--yes", action = "store_true", help = "Confirm for all dialogs." ) parser.add_argument("-c", "--check", action = "store_true", help = "Check for outdated packages." ) parser.add_argument("-l", "--latest", action = "store_true", help = "Update all packages to latest." ) parser.add_argument("-s", "--self", action = "store_true", help = "Update %s." % __name__ ) parser.add_argument("-r", "--requirements", action = "append", help = "Path(s) to requirements.txt file." ) parser.add_argument("--pipfile", action = "append", help = "Path(s) to Pipfile" ) parser.add_argument("-i", "--interactive", action = "store_true", help = "Interactive Mode" ) parser.add_argument("-p", "--project", action = "append", help = "Path(s) to Project" ) parser.add_argument("--git-username", help = "Git Username", default = getenv("GIT_USERNAME") ) parser.add_argument("--git-email", help = "Git Email", default = getenv("GIT_EMAIL") ) parser.add_argument("--pull-request", action = "store_true", help = "Perform a Pull Request" ) parser.add_argument("--github-access-token", help = "GitHub Access Token", default = getenv("GITHUB_ACCESS_TOKEN") ) parser.add_argument("--github-reponame", help = "Target GitHub Repository Name", default = getenv("GITHUB_REPONAME") ) parser.add_argument("--github-username", help = "Target GitHub Username", default = getenv("GITHUB_USERNAME") ) parser.add_argument("--target-branch", help = "Target Branch", default = getenv("TARGET_BRANCH", "master") ) parser.add_argument("-u", "--user", action = "store_true", help = "Install to the Python user install directory for environment \ variables and user configuration." ) parser.add_argument("--no-color", action = "store_true", help = "Avoid colored output." ) parser.add_argument("-V", "--verbose", action = "store_true", help = "Display verbose output." ) parser.add_argument("-v", "--version", action = "version", version = __version__, help = "Show %s's version number and exit." % __name__ ) parser.add_argument("-h", "--help", action = "help", default = argparse.SUPPRESS, help = "Show this help message and exit." ) return parser
def get_parser(): parser = argparse.ArgumentParser( prog = __command__, description = _DESCRIPTION_JUMBOTRON, add_help = False, formatter_class = ArgumentParserFormatter ) parser.add_argument("packages", nargs = "*", help = "Packages to Upgrade." ) # parser.add_argument("--resolve", # action = "store_true", # help = "Resolve Dependencies" # ) parser.add_argument("--ignore", action = "append", help = "Ignore packages to upgrade." ) parser.add_argument("--pip-path", action = "append", help = "Path to pip executable to be used." ) parser.add_argument("-y", "--yes", action = "store_true", default = getenv("ACCEPT_ALL_DIALOGS", False), help = "Confirm for all dialogs." ) parser.add_argument("-c", "--check", action = "store_true", default = getenv("DRY_RUN", False), help = "Perform a dry-run, avoid updating packages." ) parser.add_argument("--upgrade-type", choices = ("major", "minor", "patch"), nargs = "+", default = ["minor", "patch"], help = "Upgrade Type" ) parser.add_argument("-l", "--latest", action = "store_true", default = getenv("UPDATE_LATEST", False), help = "Update all packages to latest." ) parser.add_argument("-f", "--format", choices = ["table", "tree", "json", "yaml"], help = "Display packages format.", default = getenv("DISPLAY_FORMAT", "table") ) parser.add_argument("-a", "--all", action = "store_true", default = getenv("DISPLAY_ALL_PACKAGES", False), help = "List all packages." ) parser.add_argument("--pip", action = "store_true", default = getenv("UPDATE_PIP", False), help = "Update pip." ) parser.add_argument("-s", "--self", action = "store_true", help = "Update %s." % __name__ ) parser.add_argument("-r", "--requirements", action = "append", help = "Path(s) to requirements.txt file." ) parser.add_argument("--pipfile", action = "append", help = "Path(s) to Pipfile" ) parser.add_argument("-i", "--interactive", action = "store_true", default = getenv("INTERACTIVE", False), help = "Interactive Mode." ) parser.add_argument("-p", "--project", action = "append", help = "Path(s) to Project" ) parser.add_argument("--git-username", help = "Git Username", default = getenv("GIT_USERNAME") ) parser.add_argument("--git-email", help = "Git Email", default = getenv("GIT_EMAIL") ) parser.add_argument("--pull-request", action = "store_true", help = "Perform a Pull Request." ) parser.add_argument("--github-access-token", help = "GitHub Access Token", default = getenv("GITHUB_ACCESS_TOKEN") ) parser.add_argument("--github-reponame", help = "Target GitHub Repository Name", default = getenv("GITHUB_REPONAME") ) parser.add_argument("--github-username", help = "Target GitHub Username", default = getenv("GITHUB_USERNAME") ) parser.add_argument("--target-branch", help = "Target Branch", default = getenv("TARGET_BRANCH", "master") ) parser.add_argument("-j", "--jobs", type = int, help = "Number of Jobs to be used.", default = getenv("JOBS", max(mp.cpu_count(), 4)) ) parser.add_argument("-u", "--user", action = "store_true", default = getenv("USER_ONLY", False), help = "Install to the Python user install directory for environment \ variables and user configuration." ) parser.add_argument("--no-included-requirements", action = "store_true", default = getenv("NO_INCLUDED_REQUIREMENTS", False), help = "Avoid updating included requirements." ) parser.add_argument("--no-cache", action = "store_true", default = getenv("NO_CACHE", False), help = "Avoid fetching latest updates from PyPI server." ) parser.add_argument("-o", "--output", default = getenv("OUTPUT_FILE"), help = "Print Output to File." ) parser.add_argument("--ignore-error", action = "store_true", default = getenv("IGNORE_ERROR", False), help = "Ignore Error in case of upgrade failure." ) parser.add_argument("--force", action = "store_true", default = getenv("FORCE", False), help = "Force search for files within a project." ) if _CAN_ANSI_FORMAT or "pytest" in sys.modules: parser.add_argument("--no-color", action = "store_true", default = getenv("NO_COLOR", False), help = "Avoid colored output." ) parser.add_argument("-V", "--verbose", action = "store_true", help = "Display verbose output.", default = getenv("VERBOSE", False) ) parser.add_argument("-v", "--version", action = "version", version = __version__, help = "Show %s's version number and exit." % __name__ ) parser.add_argument("-h", "--help", action = "help", default = argparse.SUPPRESS, help = "Show this help message and exit." ) return parser
def run_all(): logger.info("Running all jobs...") for job in JOBS: if not job.get("beta") or getenv("JOBS_BETA"): run_job(job["name"], variables=job.get("variables"))
def test_getenv(): with pytest.raises(KeyError): assert getenv("ABCDEFGHIJKLMNOPQRSTUVWZYX", raise_err=True)
def run(*args, **kwargs): dir_path = PATH["CACHE"] # seed database... repo = osp.join(dir_path, "pipupgrade") if not osp.exists(repo): github_username = getenv("JOBS_GITHUB_USERNAME", raise_err=True) github_oauth_token = getenv("JOBS_GITHUB_OAUTH_TOKEN", raise_err=True) popen( "git clone https://%s:%[email protected]/achillesrasquinha/pipupgrade %s" % (github_username, github_oauth_token, repo), cwd=dir_path) popen("git config user.email '*****@*****.**'", cwd=repo) popen("git config user.name 'pipupgrade bot'", cwd=repo) else: try: popen("git pull origin master", cwd=repo) except PopenError: logger.warn("Unable to pull latest branch") deptree = Dict() path_deptree = osp.join(repo, "data", "dependencies.json.gz") if osp.exists(path_deptree): with gzip.open(path_deptree) as f: content = f.read() deptree = Dict(json.loads(content)) with make_temp_dir() as dir_path: chunk_size = kwargs.get("chunk_size", 1000) index_url = kwargs.get("index_url", BASE_INDEX_URL) logger.info("Fetching Package List...") res = proxy_request("GET", index_url, stream=True) res.raise_for_status() html = "" for content in res.iter_content(chunk_size=1024): html += safe_decode(content) soup = BeautifulSoup(html, 'html.parser') packages = list( filter(lambda x: x not in deptree, map(lambda x: x.text, soup.findAll('a')))) logger.info("%s packages found." % len(packages)) package_chunks = list(chunkify(packages, chunk_size)) for package_chunk in tqdm(package_chunks): requestsmap = (proxy_grequest( "GET", "https://pypi.org/pypi/%s/json" % package) for package in package_chunk) responses = grequests.map(requestsmap, exception_handler=exception_handler) for response in responses: if response.ok: data = response.json() package = data["info"]["name"] releases = list( filter(lambda x: x not in deptree[package], iterkeys(data["releases"]))) release_chunks = chunkify(releases, 100) for release_chunk in release_chunks: requestsmap = (proxy_grequest( "GET", "https://pypi.org/pypi/%s/%s/json" % (package, release)) for release in release_chunk) responses = grequests.map( requestsmap, exception_handler=exception_handler) for response in responses: if response.ok: data = response.json() version = data["info"]["version"] requires = data["info"]["requires_dist"] deptree[package][version] = requires # query = """ # INSERT OR IGNORE INTO `tabPackageDependency` # (name, version, requires) # VALUES # (?, ?, ?) # """ # values = ( # package, # version, # ",".join(requires) if requires else "NULL" # ) # connection.query(query, values) else: logger.info("Unable to load URL: %s" % response.url) else: logger.info("Unable to load URL: %s" % response.url) with gzip.open(path_deptree, mode="wt") as f: content = json.dumps(deptree) f.write(content) popen("git add %s" % path_deptree, cwd=repo) popen( "git commit --allow-empty -m '[skip ci]: Update database - %s'" % get_timestamp_str(), cwd=repo) popen("git push origin master", cwd=repo)