def popen(*args, **kwargs):
output = kwargs.get("output", False)
quiet = kwargs.get("quiet", False)
directory = kwargs.get("cwd")
environment = kwargs.get("env")
shell = kwargs.get("shell", True)
raise_err = kwargs.get("raise_err", True)
environ = os.environ.copy()
if environment:
environ.update(environment)
for k, v in iteritems(environ):
environ[k] = str(v)
command = " ".join([str(arg) for arg in args])
logger.info("Executing command: %s" % command)
if quiet:
output = True
proc = sp.Popen(command,
bufsize=-1,
stdin=sp.PIPE if output else kwargs.get("stdin"),
stdout=sp.PIPE if output else None,
stderr=sp.PIPE if output else None,
env=environ,
cwd=directory,
shell=shell)
code = proc.wait()
if code and raise_err:
print("An error occurred while updating package:",
PopenError(code, command))
#raise PopenError(code, command)
if output:
output, error = proc.communicate()
if output:
output = safe_decode(output)
output = strip(output)
if error:
error = safe_decode(error)
error = strip(error)
if quiet:
return code
else:
return code, output, error
else:
return code
def __exit__(self, *args):
self.output = safe_decode(self._output.getvalue())
del self._output
sys.stdout = self._stdout
def assert_stdout(capfd, output):
stdout, _ = capfd.readouterr()
assert safe_decode(output) == safe_decode(stdout)
def run(*args, **kwargs):
dir_path = PATH["CACHE"]
# seed database...
repo = osp.join(dir_path, "pipupgrade")
if not osp.exists(repo):
github_username = getenv("JOBS_GITHUB_USERNAME", raise_err=True)
github_oauth_token = getenv("JOBS_GITHUB_OAUTH_TOKEN", raise_err=True)
popen(
"git clone https://%s:%[email protected]/achillesrasquinha/pipupgrade %s"
% (github_username, github_oauth_token, repo),
cwd=dir_path)
popen("git config user.email '*****@*****.**'", cwd=repo)
popen("git config user.name 'pipupgrade bot'", cwd=repo)
else:
try:
popen("git pull origin master", cwd=repo)
except PopenError:
logger.warn("Unable to pull latest branch")
deptree = Dict()
path_deptree = osp.join(repo, "data", "dependencies.json.gz")
if osp.exists(path_deptree):
with gzip.open(path_deptree) as f:
content = f.read()
deptree = Dict(json.loads(content))
with make_temp_dir() as dir_path:
chunk_size = kwargs.get("chunk_size", 1000)
index_url = kwargs.get("index_url", BASE_INDEX_URL)
logger.info("Fetching Package List...")
res = proxy_request("GET", index_url, stream=True)
res.raise_for_status()
html = ""
for content in res.iter_content(chunk_size=1024):
html += safe_decode(content)
soup = BeautifulSoup(html, 'html.parser')
packages = list(
filter(lambda x: x not in deptree,
map(lambda x: x.text, soup.findAll('a'))))
logger.info("%s packages found." % len(packages))
package_chunks = list(chunkify(packages, chunk_size))
for package_chunk in tqdm(package_chunks):
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/json" % package)
for package in package_chunk)
responses = grequests.map(requestsmap,
exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
package = data["info"]["name"]
releases = list(
filter(lambda x: x not in deptree[package],
iterkeys(data["releases"])))
release_chunks = chunkify(releases, 100)
for release_chunk in release_chunks:
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/%s/json" %
(package, release)) for release in release_chunk)
responses = grequests.map(
requestsmap, exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
version = data["info"]["version"]
requires = data["info"]["requires_dist"]
deptree[package][version] = requires
# query = """
# INSERT OR IGNORE INTO `tabPackageDependency`
# (name, version, requires)
# VALUES
# (?, ?, ?)
# """
# values = (
# package,
# version,
# ",".join(requires) if requires else "NULL"
# )
# connection.query(query, values)
else:
logger.info("Unable to load URL: %s" %
response.url)
else:
logger.info("Unable to load URL: %s" % response.url)
with gzip.open(path_deptree, mode="wt") as f:
content = json.dumps(deptree)
f.write(content)
popen("git add %s" % path_deptree, cwd=repo)
popen(
"git commit --allow-empty -m '[skip ci]: Update database - %s'"
% get_timestamp_str(),
cwd=repo)
popen("git push origin master", cwd=repo)
def test_safe_decode():
assert safe_decode(b"foobar") == "foobar"
assert safe_decode( "foobar") == "foobar"
assert safe_decode(123456789) == 123456789