def __init__(self, location=PATH["CACHE"], name="config"):
config = getenv("CONFIG")
if not config:
self.name = "%s.ini" % name
self.location = location
makedirs(self.location, exist_ok=True)
else:
self.name = osp.basename(config)
self.location = osp.dirname(config)
self.config = self.read()
def get_parser():
parser = argparse.ArgumentParser(
prog = __command__,
description = _DESCRIPTION_JUMBOTRON,
add_help = False,
formatter_class = ArgumentParserFormatter
)
parser.add_argument("--pip-path",
action = "append",
help = "Path to pip executable to be used."
)
parser.add_argument("-y", "--yes",
action = "store_true",
help = "Confirm for all dialogs."
)
parser.add_argument("-c", "--check",
action = "store_true",
help = "Check for outdated packages."
)
parser.add_argument("-l", "--latest",
action = "store_true",
help = "Update all packages to latest."
)
parser.add_argument("-s", "--self",
action = "store_true",
help = "Update %s." % __name__
)
parser.add_argument("-r", "--requirements",
action = "append",
help = "Path(s) to requirements.txt file."
)
parser.add_argument("--pipfile",
action = "append",
help = "Path(s) to Pipfile"
)
parser.add_argument("-i", "--interactive",
action = "store_true",
help = "Interactive Mode"
)
parser.add_argument("-p", "--project",
action = "append",
help = "Path(s) to Project"
)
parser.add_argument("--git-username",
help = "Git Username",
default = getenv("GIT_USERNAME")
)
parser.add_argument("--git-email",
help = "Git Email",
default = getenv("GIT_EMAIL")
)
parser.add_argument("--pull-request",
action = "store_true",
help = "Perform a Pull Request"
)
parser.add_argument("--github-access-token",
help = "GitHub Access Token",
default = getenv("GITHUB_ACCESS_TOKEN")
)
parser.add_argument("--github-reponame",
help = "Target GitHub Repository Name",
default = getenv("GITHUB_REPONAME")
)
parser.add_argument("--github-username",
help = "Target GitHub Username",
default = getenv("GITHUB_USERNAME")
)
parser.add_argument("--target-branch",
help = "Target Branch",
default = getenv("TARGET_BRANCH", "master")
)
parser.add_argument("-u", "--user",
action = "store_true",
help = "Install to the Python user install directory for environment \
variables and user configuration."
)
parser.add_argument("--no-color",
action = "store_true",
help = "Avoid colored output."
)
parser.add_argument("-V", "--verbose",
action = "store_true",
help = "Display verbose output."
)
parser.add_argument("-v", "--version",
action = "version",
version = __version__,
help = "Show %s's version number and exit." % __name__
)
parser.add_argument("-h", "--help",
action = "help",
default = argparse.SUPPRESS,
help = "Show this help message and exit."
)
return parser
def get_parser():
parser = argparse.ArgumentParser(
prog = __command__,
description = _DESCRIPTION_JUMBOTRON,
add_help = False,
formatter_class = ArgumentParserFormatter
)
parser.add_argument("packages",
nargs = "*",
help = "Packages to Upgrade."
)
# parser.add_argument("--resolve",
# action = "store_true",
# help = "Resolve Dependencies"
# )
parser.add_argument("--ignore",
action = "append",
help = "Ignore packages to upgrade."
)
parser.add_argument("--pip-path",
action = "append",
help = "Path to pip executable to be used."
)
parser.add_argument("-y", "--yes",
action = "store_true",
default = getenv("ACCEPT_ALL_DIALOGS", False),
help = "Confirm for all dialogs."
)
parser.add_argument("-c", "--check",
action = "store_true",
default = getenv("DRY_RUN", False),
help = "Perform a dry-run, avoid updating packages."
)
parser.add_argument("--upgrade-type",
choices = ("major", "minor", "patch"),
nargs = "+",
default = ["minor", "patch"],
help = "Upgrade Type"
)
parser.add_argument("-l", "--latest",
action = "store_true",
default = getenv("UPDATE_LATEST", False),
help = "Update all packages to latest."
)
parser.add_argument("-f", "--format",
choices = ["table", "tree", "json", "yaml"],
help = "Display packages format.",
default = getenv("DISPLAY_FORMAT", "table")
)
parser.add_argument("-a", "--all",
action = "store_true",
default = getenv("DISPLAY_ALL_PACKAGES", False),
help = "List all packages."
)
parser.add_argument("--pip",
action = "store_true",
default = getenv("UPDATE_PIP", False),
help = "Update pip."
)
parser.add_argument("-s", "--self",
action = "store_true",
help = "Update %s." % __name__
)
parser.add_argument("-r", "--requirements",
action = "append",
help = "Path(s) to requirements.txt file."
)
parser.add_argument("--pipfile",
action = "append",
help = "Path(s) to Pipfile"
)
parser.add_argument("-i", "--interactive",
action = "store_true",
default = getenv("INTERACTIVE", False),
help = "Interactive Mode."
)
parser.add_argument("-p", "--project",
action = "append",
help = "Path(s) to Project"
)
parser.add_argument("--git-username",
help = "Git Username",
default = getenv("GIT_USERNAME")
)
parser.add_argument("--git-email",
help = "Git Email",
default = getenv("GIT_EMAIL")
)
parser.add_argument("--pull-request",
action = "store_true",
help = "Perform a Pull Request."
)
parser.add_argument("--github-access-token",
help = "GitHub Access Token",
default = getenv("GITHUB_ACCESS_TOKEN")
)
parser.add_argument("--github-reponame",
help = "Target GitHub Repository Name",
default = getenv("GITHUB_REPONAME")
)
parser.add_argument("--github-username",
help = "Target GitHub Username",
default = getenv("GITHUB_USERNAME")
)
parser.add_argument("--target-branch",
help = "Target Branch",
default = getenv("TARGET_BRANCH", "master")
)
parser.add_argument("-j", "--jobs",
type = int,
help = "Number of Jobs to be used.",
default = getenv("JOBS", max(mp.cpu_count(), 4))
)
parser.add_argument("-u", "--user",
action = "store_true",
default = getenv("USER_ONLY", False),
help = "Install to the Python user install directory for environment \
variables and user configuration."
)
parser.add_argument("--no-included-requirements",
action = "store_true",
default = getenv("NO_INCLUDED_REQUIREMENTS", False),
help = "Avoid updating included requirements."
)
parser.add_argument("--no-cache",
action = "store_true",
default = getenv("NO_CACHE", False),
help = "Avoid fetching latest updates from PyPI server."
)
parser.add_argument("-o", "--output",
default = getenv("OUTPUT_FILE"),
help = "Print Output to File."
)
parser.add_argument("--ignore-error",
action = "store_true",
default = getenv("IGNORE_ERROR", False),
help = "Ignore Error in case of upgrade failure."
)
parser.add_argument("--force",
action = "store_true",
default = getenv("FORCE", False),
help = "Force search for files within a project."
)
if _CAN_ANSI_FORMAT or "pytest" in sys.modules:
parser.add_argument("--no-color",
action = "store_true",
default = getenv("NO_COLOR", False),
help = "Avoid colored output."
)
parser.add_argument("-V", "--verbose",
action = "store_true",
help = "Display verbose output.",
default = getenv("VERBOSE", False)
)
parser.add_argument("-v", "--version",
action = "version",
version = __version__,
help = "Show %s's version number and exit." % __name__
)
parser.add_argument("-h", "--help",
action = "help",
default = argparse.SUPPRESS,
help = "Show this help message and exit."
)
return parser
def run_all():
logger.info("Running all jobs...")
for job in JOBS:
if not job.get("beta") or getenv("JOBS_BETA"):
run_job(job["name"], variables=job.get("variables"))
def test_getenv():
with pytest.raises(KeyError):
assert getenv("ABCDEFGHIJKLMNOPQRSTUVWZYX", raise_err=True)
def run(*args, **kwargs):
dir_path = PATH["CACHE"]
# seed database...
repo = osp.join(dir_path, "pipupgrade")
if not osp.exists(repo):
github_username = getenv("JOBS_GITHUB_USERNAME", raise_err=True)
github_oauth_token = getenv("JOBS_GITHUB_OAUTH_TOKEN", raise_err=True)
popen(
"git clone https://%s:%[email protected]/achillesrasquinha/pipupgrade %s"
% (github_username, github_oauth_token, repo),
cwd=dir_path)
popen("git config user.email '*****@*****.**'", cwd=repo)
popen("git config user.name 'pipupgrade bot'", cwd=repo)
else:
try:
popen("git pull origin master", cwd=repo)
except PopenError:
logger.warn("Unable to pull latest branch")
deptree = Dict()
path_deptree = osp.join(repo, "data", "dependencies.json.gz")
if osp.exists(path_deptree):
with gzip.open(path_deptree) as f:
content = f.read()
deptree = Dict(json.loads(content))
with make_temp_dir() as dir_path:
chunk_size = kwargs.get("chunk_size", 1000)
index_url = kwargs.get("index_url", BASE_INDEX_URL)
logger.info("Fetching Package List...")
res = proxy_request("GET", index_url, stream=True)
res.raise_for_status()
html = ""
for content in res.iter_content(chunk_size=1024):
html += safe_decode(content)
soup = BeautifulSoup(html, 'html.parser')
packages = list(
filter(lambda x: x not in deptree,
map(lambda x: x.text, soup.findAll('a'))))
logger.info("%s packages found." % len(packages))
package_chunks = list(chunkify(packages, chunk_size))
for package_chunk in tqdm(package_chunks):
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/json" % package)
for package in package_chunk)
responses = grequests.map(requestsmap,
exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
package = data["info"]["name"]
releases = list(
filter(lambda x: x not in deptree[package],
iterkeys(data["releases"])))
release_chunks = chunkify(releases, 100)
for release_chunk in release_chunks:
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/%s/json" %
(package, release)) for release in release_chunk)
responses = grequests.map(
requestsmap, exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
version = data["info"]["version"]
requires = data["info"]["requires_dist"]
deptree[package][version] = requires
# query = """
# INSERT OR IGNORE INTO `tabPackageDependency`
# (name, version, requires)
# VALUES
# (?, ?, ?)
# """
# values = (
# package,
# version,
# ",".join(requires) if requires else "NULL"
# )
# connection.query(query, values)
else:
logger.info("Unable to load URL: %s" %
response.url)
else:
logger.info("Unable to load URL: %s" % response.url)
with gzip.open(path_deptree, mode="wt") as f:
content = json.dumps(deptree)
f.write(content)
popen("git add %s" % path_deptree, cwd=repo)
popen(
"git commit --allow-empty -m '[skip ci]: Update database - %s'"
% get_timestamp_str(),
cwd=repo)
popen("git push origin master", cwd=repo)
