def extract_data(self):
"""Extracts data from PyPI and archive.
Returns:
PackageData object containing data extracted from PyPI and archive.
"""
try:
release_urls = self.client.release_urls(self.name, self.version)
release_data = self.client.release_data(self.name, self.version)
except: # some kind of error with client => return TODO: log the failure
logger.debug('Client: {0} Name: {1} Version: {2}.'.format(
self.client, self.name, self.version))
logger.warn('Some kind of error while communicating with client: {0}.'.format(
self.client), exc_info=True)
return PackageData(self.local_file,
self.name,
self.name_convertor.rpm_name(self.name)
if self.rpm_name is None else self.rpm_name,
self.version,
'FAILED TO EXTRACT FROM PYPI',
'FAILED TO EXTRACT FROM PYPI')
url = ''
md5_digest = None
if len(release_urls):
for release_url in release_urls:
if release_url['url'].endswith("tar.gz"):
url = release_url['url']
md5_digest = release_url['md5_digest']
if url == '':
url = release_urls[0]['url']
md5_digest = release_urls[0]['md5_digest']
elif release_data:
url = release_data['download_url']
data = PackageData(self.local_file,
self.name,
self.name_convertor.rpm_name(self.name)
if self.rpm_name is None else self.rpm_name,
self.version,
md5_digest,
url)
for data_field in settings.PYPI_USABLE_DATA:
setattr(data, data_field, release_data.get(data_field, ''))
pypi_license = data.license
with self.archive:
data.set_from(self.data_from_archive)
# for example nose has attribute `packages` but instead of name listing the
# packages is using function to find them, that makes data.packages an empty set
if data.has_packages and not data.packages:
data.packages.add(data.name)
# we usually get better license representation from trove classifiers
data.license = utils.license_from_trove(
release_data.get('classifiers', '')) or data.license or pypi_license
return data
def extract_data(self):
"""Extracts data from PyPI and archive.
Returns:
PackageData object containing data extracted from PyPI and archive.
"""
try:
release_urls = self.client.release_urls(self.name, self.version)
release_data = self.client.release_data(self.name, self.version)
except: # some kind of error with client => return TODO: log the failure
return PackageData(self.local_file,
self.name,
self.name_convertor.rpm_name(self.name) \
if self.rpm_name == None else self.rpm_name,
self.version,
'FAILED TO EXTRACT FROM PYPI',
'FAILED TO EXTRACT FROM PYPI')
url = ''
md5_digest = None
if len(release_urls) > 0:
url_bak = release_urls[0]['url']
md5_bak = release_urls[0]['md5_digest']
for release_url in release_urls:
if release_url['url'].endswith("tar.gz"):
url = release_url['url']
md5_digest = release_url['md5_digest']
elif release_url['url'].endswith("zip"):
url_bak = release_url['url']
md5_bak = release_url['md5_digest']
else:
if url == '':
url = url_bak
md5_digest = md5_bak
elif release_data:
url = release_data['download_url']
data = PackageData(self.local_file,
self.name,
self.name_convertor.rpm_name(self.name) \
if self.rpm_name == None else self.rpm_name,
self.version,
md5_digest,
url)
for data_field in settings.PYPI_USABLE_DATA:
setattr(data, data_field, release_data.get(data_field, ''))
with self.archive:
data.set_from(self.data_from_archive)
# we usually get better license representation from trove classifiers
data.license = utils.license_from_trove(release_data.get('classifiers', '')) or data.license
return data
def inner(self, client=None):
data = extraction_fce(self)
if client is None:
logger.debug("Client is None, skipping metadata extraction from PyPI.")
return data
try:
release_urls = client.release_urls(self.name, self.version)
release_data = client.release_data(self.name, self.version)
except: # some kind of error with client => return TODO: log the failure
logger.debug('Client: {0} Name: {1} Version: {2}.'.format(
client, self.name, self.version))
logger.warn('Some kind of error while communicating with client: {0}.'.format(
client), exc_info=True)
failure_dict = {'md5': 'FAILED TO EXTRACT FROM PYPI',
'url': 'FAILED TO EXTRACT FROM PYPI'}
data.set_from(failure_dict, update=True)
return data
url = ''
md5_digest = None
if len(release_urls):
zip_url = zip_md5 = ''
for release_url in release_urls:
if release_url['url'].endswith("tar.gz"):
url = release_url['url']
md5_digest = release_url['md5_digest']
if release_url['url'].endswith(".zip"):
zip_url = release_url['url']
zip_md5 = release_url['md5_digest']
if url == '':
url = zip_url or release_urls[0]['url']
md5_digest = zip_md5 or release_urls[0]['md5_digest']
elif release_data:
url = release_data['download_url']
data_dict = {'md5': md5_digest, 'url': url}
for data_field in settings.PYPI_USABLE_DATA:
data_dict[data_field] = release_data.get(data_field, '')
# we usually get better license representation from trove classifiers
data_dict["license"] = utils.license_from_trove(release_data.get('classifiers', ''))
data.set_from(data_dict, update=True)
return data
def inner(self, client=None):
data = extraction_fce(self)
try:
if client is None:
raise ValueError("Client is None.")
release_data = client.release_data(self.name, self.version)
except:
logger.warning('Some kind of error while communicating with client: {0}.'.format(
client), exc_info=True)
return data
url, md5_digest = get_url(client, self.name, self.version)
data_dict = {'url': url, 'md5': md5_digest}
for data_field in settings.PYPI_USABLE_DATA:
data_dict[data_field] = release_data.get(data_field, '')
# we usually get better license representation from trove classifiers
data_dict["license"] = utils.license_from_trove(release_data.get('classifiers', ''))
data.set_from(data_dict, update=True)
return data
def inner(self, client=None):
data = extraction_fce(self)
if client is None:
logger.warning("Client is None, it was probably disabled")
return data
try:
release_data = client.release_data(self.name, self.version)
except:
logger.warning('Some kind of error while communicating with client: {0}.'.format(
client), exc_info=True)
return data
try:
url, md5_digest = get_url(client, self.name, self.version)
except exc.MissingUrlException:
url, md5_digest = ('FAILED TO EXTRACT FROM PYPI', 'FAILED TO EXTRACT FROM PYPI')
data_dict = {'url': url, 'md5': md5_digest}
for data_field in settings.PYPI_USABLE_DATA:
data_dict[data_field] = release_data.get(data_field, '')
# we usually get better license representation from trove classifiers
data_dict["license"] = utils.license_from_trove(release_data.get('classifiers', ''))
data.set_from(data_dict, update=True)
return data
def inner(self, client=None):
data = extraction_fce(self)
if client is None:
logger.warning("Client is None, it was probably disabled")
return data
try:
release_data = client.release_data(self.name, self.version)
except:
logger.warning('Some kind of error while communicating with client: {0}.'.format(
client), exc_info=True)
return data
try:
url, md5_digest = get_url(client, self.name, self.version)
except exc.MissingUrlException:
url, md5_digest = ('FAILED TO EXTRACT FROM PYPI', 'FAILED TO EXTRACT FROM PYPI')
data_dict = {'url': url, 'md5': md5_digest}
for data_field in settings.PYPI_USABLE_DATA:
data_dict[data_field] = release_data.get(data_field, '')
# we usually get better license representation from trove classifiers
data_dict["license"] = utils.license_from_trove(release_data.get('classifiers', ''))
data.set_from(data_dict, update=True)
return data
def inner(self, client=None):
data = extraction_fce(self)
try:
if client is None:
raise ValueError("Client is None.")
release_data = client.release_data(self.name, self.version)
except:
logger.warning(
'Some kind of error while communicating with client: {0}.'.
format(client),
exc_info=True)
return data
url, md5_digest = get_url(client, self.name, self.version)
data_dict = {'url': url, 'md5': md5_digest}
for data_field in settings.PYPI_USABLE_DATA:
data_dict[data_field] = release_data.get(data_field, '')
# we usually get better license representation from trove classifiers
data_dict["license"] = utils.license_from_trove(
release_data.get('classifiers', ''))
data.set_from(data_dict, update=True)
return data
def license_from_egg_info(self):
return utils.license_from_trove(self.archive.get_content_of_file(
'EGG-INFO/PKG-INFO', True).splitlines())
def license_from_setup_py(self):
return utils.license_from_trove(self.archive.find_list_argument('classifiers'))
def test_license_from_trove(self, input, expected):
assert utils.license_from_trove(input) == expected
def license_from_egg_info(self):
return utils.license_from_trove(self.archive.get_content_of_file(
'EGG-INFO/PKG-INFO', True).splitlines())
def license_from_setup_py(self):
return utils.license_from_trove(self.archive.find_list_argument('classifiers'))
def test_license_from_trove(self, input, expected):
assert utils.license_from_trove(input) == expected
def license_from_setup_py(self):
license = utils.license_from_trove(self.archive.find_list_argument('classifiers'))
if not license:
license = self.archive.find_string_argument('license')
return license
def extract_data(self):
"""Extracts data from PyPI and archive.
Returns:
PackageData object containing data extracted from PyPI and archive.
"""
try:
release_urls = self.client.release_urls(self.name, self.version)
release_data = self.client.release_data(self.name, self.version)
except: # some kind of error with client => return TODO: log the failure
logger.debug("Client: {0} Name: {1} Version: {2}.".format(self.client, self.name, self.version))
logger.warn("Some kind of error while communicating with client: {0}.".format(self.client), exc_info=True)
return PackageData(
self.local_file,
self.name,
self.name_convertor.rpm_name(self.name) if self.rpm_name is None else self.rpm_name,
self.version,
"FAILED TO EXTRACT FROM PYPI",
"FAILED TO EXTRACT FROM PYPI",
)
url = ""
md5_digest = None
if len(release_urls):
for release_url in release_urls:
if release_url["url"].endswith("tar.gz"):
url = release_url["url"]
md5_digest = release_url["md5_digest"]
if url == "":
url = release_urls[0]["url"]
md5_digest = release_urls[0]["md5_digest"]
elif release_data:
url = release_data["download_url"]
data = PackageData(
self.local_file,
self.name,
self.name_convertor.rpm_name(self.name) if self.rpm_name is None else self.rpm_name,
self.version,
md5_digest,
url,
)
for data_field in settings.PYPI_USABLE_DATA:
setattr(data, data_field, release_data.get(data_field, ""))
pypi_license = data.license
with self.archive:
data.set_from(self.data_from_archive)
data.set_from(self.data_from_venv, update=True)
setattr(data, "scripts", utils.remove_major_minor_suffix(data.data["scripts"]))
data.data["build_deps"] += utils.runtime_to_build(data.data["runtime_deps"])
setattr(data, "build_deps", utils.unique_deps(data.data["build_deps"]))
# Append all runtime deps to build deps and unique the result
# for example nose has attribute `packages` but instead of name listing the
# packages is using function to find them, that makes data.packages an empty set
if data.has_packages and not data.packages:
data.packages.add(data.name)
# we usually get better license representation from trove classifiers
data.license = utils.license_from_trove(release_data.get("classifiers", "")) or data.license or pypi_license
return data