def test_iterkeys():
dict_ = dict(foo="bar")
assert isinstance(iterkeys(dict_), collections.Iterable)
for k in iterkeys(dict_):
assert k in dict_
def foobar(*args, **kwargs):
args = get_args()
params = merge_dict(args, override)
for k, v in iteritems(values):
assert params[k] == v
if initial:
for k in iterkeys(initial):
assert initial[k] == args[k]
def run(*args, **kwargs):
dir_path = PATH["CACHE"]
# seed database...
repo = osp.join(dir_path, "pipupgrade")
if not osp.exists(repo):
github_username = getenv("JOBS_GITHUB_USERNAME", raise_err=True)
github_oauth_token = getenv("JOBS_GITHUB_OAUTH_TOKEN", raise_err=True)
popen(
"git clone https://%s:%[email protected]/achillesrasquinha/pipupgrade %s"
% (github_username, github_oauth_token, repo),
cwd=dir_path)
popen("git config user.email '*****@*****.**'", cwd=repo)
popen("git config user.name 'pipupgrade bot'", cwd=repo)
else:
try:
popen("git pull origin master", cwd=repo)
except PopenError:
logger.warn("Unable to pull latest branch")
deptree = Dict()
path_deptree = osp.join(repo, "data", "dependencies.json.gz")
if osp.exists(path_deptree):
with gzip.open(path_deptree) as f:
content = f.read()
deptree = Dict(json.loads(content))
with make_temp_dir() as dir_path:
chunk_size = kwargs.get("chunk_size", 1000)
index_url = kwargs.get("index_url", BASE_INDEX_URL)
logger.info("Fetching Package List...")
res = proxy_request("GET", index_url, stream=True)
res.raise_for_status()
html = ""
for content in res.iter_content(chunk_size=1024):
html += safe_decode(content)
soup = BeautifulSoup(html, 'html.parser')
packages = list(
filter(lambda x: x not in deptree,
map(lambda x: x.text, soup.findAll('a'))))
logger.info("%s packages found." % len(packages))
package_chunks = list(chunkify(packages, chunk_size))
for package_chunk in tqdm(package_chunks):
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/json" % package)
for package in package_chunk)
responses = grequests.map(requestsmap,
exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
package = data["info"]["name"]
releases = list(
filter(lambda x: x not in deptree[package],
iterkeys(data["releases"])))
release_chunks = chunkify(releases, 100)
for release_chunk in release_chunks:
requestsmap = (proxy_grequest(
"GET", "https://pypi.org/pypi/%s/%s/json" %
(package, release)) for release in release_chunk)
responses = grequests.map(
requestsmap, exception_handler=exception_handler)
for response in responses:
if response.ok:
data = response.json()
version = data["info"]["version"]
requires = data["info"]["requires_dist"]
deptree[package][version] = requires
# query = """
# INSERT OR IGNORE INTO `tabPackageDependency`
# (name, version, requires)
# VALUES
# (?, ?, ?)
# """
# values = (
# package,
# version,
# ",".join(requires) if requires else "NULL"
# )
# connection.query(query, values)
else:
logger.info("Unable to load URL: %s" %
response.url)
else:
logger.info("Unable to load URL: %s" % response.url)
with gzip.open(path_deptree, mode="wt") as f:
content = json.dumps(deptree)
f.write(content)
popen("git add %s" % path_deptree, cwd=repo)
popen(
"git commit --allow-empty -m '[skip ci]: Update database - %s'"
% get_timestamp_str(),
cwd=repo)
popen("git push origin master", cwd=repo)
def __init__(self, package, sync=False, pip_exec=None):
logger.info("Initializing Package %s of type %s..." %
(package, type(package)))
self.current_version = None
# self.dependencies = [ ]
if isinstance(package, (_pip.Distribution, _pip.DistInfoDistribution,
_pip.EggInfoDistribution)):
self.name = package.project_name
self.current_version = package.version
elif isinstance(package, _pip.InstallRequirement):
self.name = package.name
if hasattr(package, "req"):
if hasattr(package.req, "specifier"):
self.current_version = str(package.req.specifier)
else:
self.current_version = package.installed_version
elif isinstance(package, dict):
self.name = package["name"]
self.current_version = package["version"]
self.latest_version = package.get("latest_version")
elif isinstance(package, str):
self.name = package
if pip_exec:
info = _get_pip_info(self.name)
self.current_version = info["version"]
# self.dependencies = info["requires"]
# if pip_exec and not self.dependencies:
# self.dependencies = _get_pip_info(self.name)
res = None
try:
logger.info("Fetching package %s information from DB..." %
self.name)
res = _db.query("""
SELECT *
FROM `tabPackage`
WHERE name = '%s'
""" % self.name)
except db.OperationalError as e:
logger.warn("Unable to fetch package name. %s" % e)
if res:
cache_timeout = settings.get("cache_timeout")
if res["_updated_at"]:
time_difference = to_datetime(
res["_updated_at"]) + timedelta(seconds=cache_timeout)
if datetime.now() > time_difference:
sync = True
else:
sync = True
if not res or sync:
logger.info("Fetching PyPI info for package %s..." % self)
_pypi_info = _get_pypi_info(self.name, raise_err=False) or {}
if not getattr(self, "latest_version", None) or sync:
self.latest_version = _pypi_info.get("version")
self.home_page = _pypi_info.get("home_page")
self.releases = [
version for version in iterkeys(_pypi_info.get("releases"))
]
if not res:
try:
values = (self.name, self.latest_version
or "NULL", self.home_page, ",".join(self.releases),
datetime.now(), datetime.now())
logger.info(
"Attempting to INSERT package %s into database with values: %s."
% (self, values))
_db.query("""
INSERT INTO `tabPackage`
(name, latest_version, home_page, releases, _created_at, _updated_at)
VALUES
('%s', '%s', '%s', '%s', '%s', '%s')
""" % values)
except (db.IntegrityError, db.OperationalError) as e:
logger.warn("Unable to save package name. %s" % e)
else:
if sync:
logger.info(
"Attempting to UPDATE package %s within database." % self)
try:
_db.query("""
UPDATE `tabPackage`
SET latest_version = '%s', home_page = '%s', releases = '%s', _updated_at = '%s'
WHERE
name = '%s'
""" % (self.latest_version, self.home_page, ",".join(
self.releases), datetime.now(), self.name))
except db.OperationalError as e:
logger.warn("Unable to update package name. %s" % e)
else:
logger.info("Using cached info for package %s." % self)
self.latest_version = res["latest_version"]
self.home_page = res["home_page"]
self.releases = res["releases"].split(",")
self.dependency_tree = TreeNode(self)
