def run(mode, argument):
logging.info('-'*30)
logging.info('Checking ' + argument)
if mode == 'directory':
data = projectdata.get_data(os.path.abspath(argument))
logging.info('Found ' + data.get('name', 'nothing'))
elif mode == 'file':
data = distributiondata.get_data(os.path.abspath(argument))
logging.info('Found ' + data.get('name', 'nothing'))
else:
# It's probably a package name
data = pypidata.get_data(argument)
logging.info('Found ' + data.get('name', 'nothing'))
rating = ratings.rate(data)
logging.info('-'*30)
for problem in rating[1]:
# XXX It would be nice with a * pointlist instead, but that requires
# that we know how wide the terminal is and nice word-breaks, so that's
# for later.
logging.info(problem)
if rating[1]:
logging.info('-'*30)
logging.info('Final rating: ' + str(rating[0]) + '/10')
logging.info(ratings.LEVELS[rating[0]])
logging.info('-'*30)
return rating[0]
def run(mode, argument):
logging.info('-'*30)
logging.info('Checking ' + argument)
if mode == 'directory':
data = projectdata.get_data(os.path.abspath(argument))
logging.info('Found ' + data.get('name', 'nothing'))
elif mode == 'file':
data = distributiondata.get_data(os.path.abspath(argument))
logging.info('Found ' + data.get('name', 'nothing'))
else:
# It's probably a package name
data = pypidata.get_data(argument)
logging.info('Found ' + data.get('name', 'nothing'))
rating = ratings.rate(data)
logging.info('-'*30)
for problem in rating[1]:
# XXX It would be nice with a * pointlist instead, but that requires
# that we know how wide the terminal is and nice word-breaks, so that's
# for later.
logging.info(problem)
if rating[1]:
logging.info('-'*30)
logging.info('Final rating: ' + str(rating[0]) + '/10')
logging.info(ratings.LEVELS[rating[0]])
logging.info('-'*30)
return rating[0]
def get_data(project):
client = _get_client()
# Pick the latest release.
releases = client.package_releases(project)
if not releases:
# Try to find project by case-insensitive name
project_name = project.lower()
projects = client.search({'name': project_name})
projects = [p for p in projects if p['name'].lower() == project_name]
if not projects:
raise ValueError(
"Did not find '%s' on PyPI. Did you misspell it?" % project)
project = projects[0]['name']
releases = [p['version'] for p in reversed(projects)]
release = releases[0]
# Get the metadata:
logging.debug("Found %s version %s" % (project, release))
data = client.release_data(project, release)
# Map things around:
data['long_description'] = data['description']
data['description'] = data['summary']
roles = client.package_roles(project)
data['_owners'] = [user for (role, user) in roles if role == 'Owner']
# Get download_urls:
urls = client.release_urls(project, release)
data['_pypi_downloads'] = bool(urls)
# If there is a source download, download it, and get that data.
# This is done mostly to do the imports check.
data['_source_download'] = False
data['_setuptools'] = None # Mark as unknown, in case no sdist is found.
data['_has_sdist'] = False
for download in urls:
if download['packagetype'] == 'sdist':
# Found a source distribution. Download and analyze it.
data['_has_sdist'] = True
tempdir = tempfile.gettempdir()
filename = download['url'].split('/')[-1]
tmp = os.path.join(tempdir, filename)
logging.debug("Downloading %s to verify distribution" % filename)
try:
with open(tmp, 'wb') as outfile:
outfile.write(urllib.urlopen(download['url']).read())
ddata = distributiondata.get_data(tmp)
except Exception:
# Clean up the file
os.unlink(tmp)
raise
# Combine them, with the PyPI data winning:
ddata.update(data)
data = ddata
data['_source_download'] = True
break
return data
def test_complete(self):
directory = resource_filename(__name__, os.path.join("testdata", "distributions"))
for filename in os.listdir(directory):
if filename.startswith("complete"):
data = distributiondata.get_data(os.path.join(directory, filename))
self.assertEqual(data, COMPLETE)
def run(mode, argument):
logging.info("-" * 30)
logging.info("Checking " + argument)
if mode == "directory":
data = projectdata.get_data(os.path.abspath(argument))
logging.info("Found " + data.get("name", "nothing"))
elif mode == "file":
data = distributiondata.get_data(os.path.abspath(argument))
logging.info("Found " + data.get("name", "nothing"))
else:
# It's probably a package name
data = pypidata.get_data(argument)
logging.info("Found " + data.get("name", "nothing"))
rating = ratings.rate(data)
logging.info("-" * 30)
for problem in rating[1]:
# XXX It would be nice with a * pointlist instead, but that requires
# that we know how wide the terminal is and nice word-breaks, so that's
# for later.
logging.info(problem)
if rating[1]:
logging.info("-" * 30)
logging.info("Final rating: " + str(rating[0]) + "/10")
logging.info(ratings.LEVELS[rating[0]])
logging.info("-" * 30)
return rating[0]
def get_data(project):
client = _get_client()
# Pick the latest release.
releases = client.package_releases(project)
if not releases:
# Try to find project by case-insensitive name
project_name = project.lower()
projects = client.search({"name": project_name})
projects = [p for p in projects if p["name"].lower() == project_name]
if not projects:
raise ValueError(
f"Did not find '{project}' on PyPI. Did you misspell it?")
project = projects[0]["name"]
releases = [p["version"] for p in reversed(projects)]
release = releases[0]
# Get the metadata:
logging.debug(f"Found {project} version {release}")
data = client.release_data(project, release)
# Map things around:
data["long_description"] = data["description"]
data["description"] = data["summary"]
roles = client.package_roles(project)
data["_owners"] = [user for (role, user) in roles if role == "Owner"]
# Get download_urls:
urls = client.release_urls(project, release)
data["_pypi_downloads"] = bool(urls)
# If there is a source download, download it, and get that data.
# This is done mostly to do the imports check.
data["_source_download"] = False
data["_setuptools"] = None # Mark as unknown, in case no sdist is found.
data["_has_sdist"] = False
for download in urls:
if download["packagetype"] == "sdist":
# Found a source distribution. Download and analyze it.
data["_has_sdist"] = True
tempdir = tempfile.gettempdir()
filename = download["url"].split("/")[-1]
tmp = os.path.join(tempdir, filename)
logging.debug(f"Downloading {filename} to verify distribution")
try:
with open(tmp, "wb") as outfile:
outfile.write(urllib.urlopen(download["url"]).read())
ddata = distributiondata.get_data(tmp)
except Exception:
# Clean up the file
os.unlink(tmp)
raise
# Combine them, with the PyPI data winning:
ddata.update(data)
data = ddata
data["_source_download"] = True
break
return data
def get_data(project):
# Pick the latest release.
project_data = _get_project_data(project)
releases = project_data["releases"]
data = project_data["info"]
release = data["version"]
logging.debug(f"Found {project} version {release}")
# Map things around:
data["long_description"] = data["description"]
data["description"] = data["summary"]
with xmlrpc.client.ServerProxy("https://pypi.org/pypi") as xmlrpc_client:
roles = xmlrpc_client.package_roles(project)
data["_owners"] = [user for (role, user) in roles if role == "Owner"]
# Get download_urls:
urls = releases[release]
data["_pypi_downloads"] = bool(urls)
# If there is a source download, download it, and get that data.
# This is done mostly to do the imports check.
data["_source_download"] = False
data["_setuptools"] = None # Mark as unknown, in case no sdist is found.
data["_has_sdist"] = False
for download in urls:
if download["packagetype"] == "sdist":
# Found a source distribution. Download and analyze it.
data["_has_sdist"] = True
tempdir = tempfile.gettempdir()
filename = download["url"].split("/")[-1]
tmp = os.path.join(tempdir, filename)
logging.debug(f"Downloading {filename} to verify distribution")
try:
with open(tmp, "wb") as outfile:
outfile.write(requests.get(download["url"]).content)
ddata = distributiondata.get_data(tmp)
except Exception:
# Clean up the file
os.unlink(tmp)
raise
# Combine them, with the PyPI data winning:
ddata.update(data)
data = ddata
data["_source_download"] = True
break
return data
def get_data(project):
client = _get_client()
# Pick the latest release.
releases = client.package_releases(project)
if not releases:
# Try to find project by case-insensitive name
project_name = project.lower()
projects = client.search({'name': project_name})
projects = [p for p in projects if p['name'].lower() == project_name]
if not projects:
raise ValueError(
"Did not find '%s' on PyPI. Did you misspell it?" % project)
project = projects[0]['name']
releases = [p['version'] for p in reversed(projects)]
release = releases[0]
# Get the metadata:
logging.debug("Found %s version %s" % (project, release))
data = client.release_data(project, release)
# Map things around:
data['long_description'] = data['description']
data['description'] = data['summary']
# Get download_urls:
urls = client.release_urls(project, release)
data['_pypi_downloads'] = bool(urls)
# Scrape the PyPI project page for owner info:
url = '/'.join(('http://pypi.python.org/pypi', project, release))
page = urllib.urlopen(url)
content_type = page.headers.get('content-type', '')
if '=' not in content_type:
encoding = 'utf-8'
else:
encoding = content_type.split('=')[1]
html = page.read().decode(encoding)
owners = OWNER_RE.search(html).groups()[0]
data['_owners'] = [x.strip() for x in owners.split(',')]
logging.debug("Looking for documentation")
# See if there is any docs on readthedocs
data['_readthe_docs'] = False
#import pdb;pdb.set_trace()
rtdocs = READTHEDOCS_RE.search(html)
if rtdocs:
page = urllib.urlopen(rtdocs.groups()[0])
if page.code == 200:
data['_readthe_docs'] = True
# If there is a source download, download it, and get that data.
# This is done mostly to do the imports check.
data['_source_download'] = False
data['_setuptools'] = None # Mark as unknown, in case no sdist is found.
data['_has_sdist'] = False
for download in urls:
if download['packagetype'] == 'sdist':
# Found a source distribution. Download and analyze it.
data['_has_sdist'] = True
tempdir = tempfile.gettempdir()
filename = download['url'].split('/')[-1]
tmp = os.path.join(tempdir, filename)
logging.debug("Downloading %s to verify distribution" % filename)
try:
with open(tmp, 'wb') as outfile:
outfile.write(urllib.urlopen(download['url']).read())
ddata = distributiondata.get_data(tmp)
except Exception:
# Clean up the file
os.unlink(tmp)
raise
# Combine them, with the PyPI data winning:
ddata.update(data)
data = ddata
data['_source_download'] = True
break
return data
def get_data(project):
client = _get_client()
# Pick the latest release.
releases = client.package_releases(project)
if not releases:
# Try to find project by case-insensitive name
project_name = project.lower()
projects = client.search({'name': project_name})
projects = [p for p in projects if p['name'].lower() == project_name]
if not projects:
raise ValueError(
"Did not find '%s' on PyPI. Did you misspell it?" % project)
project = projects[0]['name']
releases = [p['version'] for p in reversed(projects)]
release = releases[0]
# Get the metadata:
logging.debug("Found %s version %s" % (project, release))
data = client.release_data(project, release)
# Map things around:
data['long_description'] = data['description']
data['description'] = data['summary']
# Get download_urls:
urls = client.release_urls(project, release)
data['_pypi_downloads'] = bool(urls)
# Scrape the PyPI project page for owner info:
url = '/'.join(('http://pypi.python.org/pypi', project, release))
page = urllib.urlopen(url)
content_type = page.headers.get('content-type', '')
if '=' not in content_type:
encoding = 'utf-8'
else:
encoding = content_type.split('=')[1]
html = page.read().decode(encoding)
owners = OWNER_RE.search(html).groups()[0]
data['_owners'] = [x.strip() for x in owners.split(',')]
logging.debug("Looking for documentation")
# See if there is any docs on http://pythonhosted.org
data['_packages_docs'] = False
try:
page = urllib.urlopen('http://pythonhosted.org/' + project)
if page.code == 200:
data['_packages_docs'] = True
except urllib.HTTPError:
pass
# Maybe on readthedocs?
data['_readthe_docs'] = False
rtdocs = READTHEDOCS_RE.search(html)
if rtdocs:
page = urllib.urlopen(rtdocs.groups()[0])
if page.code == 200:
data['_readthe_docs'] = True
# If there is a source download, download it, and get that data.
# This is done mostly to do the imports check.
data['_source_download'] = False
data['_setuptools'] = None # Mark as unknown, in case no sdist is found.
data['_has_sdist'] = False
for download in urls:
if download['packagetype'] == 'sdist':
# Found a source distribution. Download and analyze it.
data['_has_sdist'] = True
tempdir = tempfile.gettempdir()
filename = download['url'].split('/')[-1]
tmp = os.path.join(tempdir, filename)
logging.debug("Downloading %s to verify distribution" % filename)
try:
with open(tmp, 'wb') as outfile:
outfile.write(urllib.urlopen(download['url']).read())
ddata = distributiondata.get_data(tmp)
except Exception:
# Clean up the file
os.unlink(tmp)
raise
# Combine them, with the PyPI data winning:
ddata.update(data)
data = ddata
data['_source_download'] = True
break
return data
