def __init__(self):
self.client = Bintray()
self.packages = load_packages()
self.existing_repos = [
r['repo']['bintray'] for p in self.packages for r in p['recipies']
]
self.missing_packages = []
self.missing_repos = []
def transform_packages(self, development=False):
""""""
github = get_github_client(3)
# Used to calculate the resources used
rate_before = github.rate_limiting
try:
with Cache(os.getenv('CACHE_FILE'), notimeout=development):
# Collect package files
packages = [os.path.join(self.packages_dir, f)
for f in os.listdir(self.packages_dir)
if os.path.isfile(os.path.join(self.packages_dir, f))]
with ThreadPoolExecutor(len(packages)) as executor:
# Generate for all packages
futures = [executor.submit(transform_package, package) for package in packages]
for _ in tqdm(as_completed(futures),
total=len(futures), unit='package', unit_scale=True,
leave=True, position=0, ncols=80):
pass
# Get results from futures and filter/print errors
results = [f.result() for f in futures]
errors = [r for r in results if isinstance(r, tuple)]
for error in errors:
exc = error[1]
print('Error in {}: {}\n{}'.format(
error[0], str(exc),
''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))))
if len(errors) > 0 and not development:
print('Errors in {} packages have prevented writing of packages.js'.format(len(errors)))
sys.exit(1)
elif len(errors) > 0 and development:
print('Errors in {} packages prevented them from being generated'.format(len(errors)))
# Write new package files
data = json.dumps([r for r in results if r is not None], indent=2)
with open('packages.json', 'w') as file:
file.write(data)
with open('packages.js', 'w') as file:
file.write('var packages_data = \n')
file.write(data)
file.write(';')
print('All generation steps succeeded and package data written')
finally:
# remaining is tuple 0, limit is tuple 1, see https://github.com/PyGithub/PyGithub/commit/ca974699b0ea2a770e6c2dbd162b3d2c0ae9fe89
rate = github.rate_limiting
print('Github rate limiting:\n\tRemaining: {}/{}\n\tUsed this call: {}'.format(
rate[0], rate[1], rate_before[0] - rate[0]))
# \n\tResets: {} (rate.reset + datetime.timedelta(hours=1)) - datetime.datetime.now()))
# PyGitHub doesn't implement the rate limit API fully right now
# see https://github.com/PyGithub/PyGithub/issues/26
bt = Bintray()
print('Bintray rate limiting:\n\tUsed this call: {}'.format(bt.rate_used))
class BintrayTransformer(BaseHTTPTransformer):
def __init__(self):
super().__init__()
self.bt = Bintray()
self.licenses = self.bt.load_licenses()
def license(self, name):
return next((l for l in self.licenses if l['name'] == name), None)
def transform(self, package):
for recipie in package.recipies:
if 'repo' in recipie and 'bintray' in recipie.repo:
bt_package = self.bt.get('/packages/' + recipie.repo.bintray)
if 'linked_to_repos' in bt_package and bt_package['linked_to_repos'] and 'conan-center' in bt_package['linked_to_repos']:
# if the package has been linked to conan-center that repository "replaces" the current one
recipie.repo.bintray = 'conan/conan-center/' + recipie.repo.bintray.split('/')[-1]
# we don't actually have to download the package information from conan-center because it will be
# identical to what we got in the "source" repository
parts = recipie.repo.bintray.split('/')
self._set_unless_exists(recipie, 'remote',
'https://api.bintray.com/conan/' + '/'.join(parts[:2]))
self._set_unless_exists(recipie, 'package', parts[2].split(':')[0])
if ':' in parts[2]:
self._set_unless_exists(recipie, 'user', parts[2].split(':')[1])
versions = [DotMap(name=v.split(':')[0],
channel=v.split(':')[1],
repo=recipie.repo.bintray,
remote=recipie.remote,
package=recipie.package,
user=recipie.user)
for v in bt_package['versions']]
if 'versions' not in package:
package.versions = []
package.versions.extend(versions)
package.versions = natsorted(package.versions, key=lambda v: v.name, reverse=True)
sites = []
if 'vcs_url' in bt_package and bt_package['vcs_url'] is not None:
sites.append(bt_package['vcs_url'])
if 'website_url' in bt_package and bt_package['website_url'] is not None:
sites.append(bt_package['website_url'])
sites = [s for s in sites if '/conan-' not in s]
for site in sites:
if 'github.com' in site:
path = site.split('/')[-2:]
if not path[1].startswith('conan') and not path[1].endswith('conan') and path[1]:
self._set_unless_exists(package.urls, 'github', '/'.join(path))
else:
self._set_unless_exists(recipie.urls, 'github', '/'.join(path))
else:
self._set_unless_exists(package.urls, 'website', site)
if 'issue_tracker_url' in bt_package and bt_package['issue_tracker_url'] is not None:
url = bt_package['issue_tracker_url']
if 'github.com/bincrafters/' not in url:
self._set_unless_exists(package.urls, 'issues', url)
if 'desc' in bt_package and bt_package['desc']:
desc = bt_package['desc']
if 'conan' in desc.lower() and ('package' in desc.lower() or 'recipe' in desc.lower()):
pass # Many descriptions on bintray are in the form "Conan package for..."
else:
self._set_unless_exists(package, 'description', bt_package['desc'])
if 'labels' in bt_package:
if 'keywords' not in package:
package.keywords = []
package.keywords.extend(bt_package['labels'])
if 'licenses' in bt_package:
self._set_unless_exists(package, 'licenses',
[self.license(l) for l in bt_package['licenses']])
if 'name' in bt_package:
self._set_unless_exists(package, 'name', bt_package['name'].split(':')[0])
files = self.bt.get('/packages/' + recipie.repo.bintray + '/files')
conanfile = next((f for f in files if f['name'] == 'conanfile.py'), None)
if conanfile is not None:
self._set_unless_exists(package.files.conanfile, 'url',
self.bt.download_url('/'.join(parts[:2]),
conanfile['path']))
self._set_unless_exists(package.files.conanfile, 'content',
self.http.get(package.files.conanfile.url).text)
return package
def __init__(self):
super().__init__()
self.bt = Bintray()
self.licenses = self.bt.load_licenses()
class BintrayFinder:
"""Class to detect, print and generate stubs for Bintray packages"""
def __init__(self):
self.client = Bintray()
self.packages = load_packages()
self.existing_repos = [
r['repo']['bintray'] for p in self.packages for r in p['recipies']
]
self.missing_packages = []
self.missing_repos = []
def _gather_repos(self) -> Set[BintrayRepoDescriptor]:
"""
Returns all repositories (tuple of repository owner and repository name) that are currently known
"""
repos = set()
for pkg in self.packages:
for recipie in pkg['recipies']:
parts = recipie['repo']['bintray'].split('/')
repos.add(BintrayRepoDescriptor(parts[0], parts[1]))
repos.add(BintrayRepoDescriptor('conan', 'conan-center'))
# Github repos that need to be created or located on bintray:
# https://github.com/syslandscape/syslandscape
# https://github.com/cajun-code/sfml.conan
# https://github.com/erikvalkering/smartref
# https://github.com/malwoden/conan-librsync
# https://github.com/LighthouseJ/mtn
# https://github.com/raulbocanegra/utils
# https://github.com/Aquaveo/xmscore
# https://github.com/franc0is/conan-jansson
# https://github.com/srand/stdext-uuid
# https://github.com/db4/conan-jwasm
# https://github.com/ulricheck/conan-librealsense
# https://github.com/iblis-ms/conan_gmock
# https://github.com/zacklj89/conan-CppMicroServices
# https://github.com/nickbruun/conan-hayai
# https://github.com/JavierJF/TabletMode
# https://github.com/google/fruit
# https://github.com/StiventoUser/conan-sqlpp11-connector-postgresql
# https://github.com/wumuzi520/conan-snappy
# https://github.com/DigitalInBlue/Celero-Conan
# https://github.com/Trick-17/FMath
# https://github.com/kwint/conan-gnuplot-iostream
# https://github.com/luisnuxx/formiga
# https://github.com/gustavokretzer88/conan-OATH-Toolkit
# https://github.com/lightningcpp/lightningcpp
# https://github.com/jasonbot/conan-jansson
# https://github.com/bfierz/vcl
# https://github.com/SverreEplov/conan-wt
# https://github.com/jampio/jkpak
# https://github.com/StableTec/vulkan
# https://github.com/tmadden/alia
# https://github.com/JavierJF/CloseApp
# https://github.com/e3dskunkworks/conan-celero
# https://github.com/kheaactua/conan-ffi
# https://github.com/matlo607/conan-swig
# https://github.com/Exiv2/exiv2
# https://github.com/Ubitrack/component_vision_aruco
# https://github.com/iblis-ms/conan_gbenchmark
# https://github.com/p47r1ck7541/conan-llvm-60
# https://github.com/StableCoder/conan-mariadb-connector
# https://github.com/yackey/AvtecGmock # owner exists
# https://github.com/yackey/AEF-CI # owner exists
# https://github.com/spielhuus/avcpp # owner exists
# https://github.com/nwoetzel/conan-omniorb # owner exists
for a in [
'inexorgame/inexor-conan', 'odant/conan', 'vuo/conan',
'squawkcpp/conan-cpp', 'degoodmanwilson/opensource',
'objectx/conan', 'franc0is/conan', 'boujee/conan', 'mdf/2out',
'impsnldavid/public-conan', 'sunxfancy/common', 'jacmoe/Conan',
'madebr/openrw', 'ogs/conan', 'danimtb/public-conan',
'kwallner/stable', 'kwallner/testing', 'dobiasd/public-conan',
'solvingj/public-conan', 'slyrisorg/SlyrisOrg',
'cliutils/CLI11', 'tuncb/pangea', 'mikayex/conan-packages',
'kenfred/conan-corner', 'p-groarke/conan-public',
'jabaa/Conan', 'blosc/Conan', 'sourcedelica/public-conan',
'sourcedelica/conan', 'rockdreamer/throwing_ptr',
'qtproject/conan', 'rhard/conan', 'tao-cpp/tao-cpp',
'bitprim/bitprim', 'yadoms/yadoms', 'enhex/enhex',
'jilabinc/conan', 'bisect/bisect', 'artalus/conan-public',
'alekseevx/pft', 'mbedded-ninja/LinuxCanBus',
'cynarakrewe/CynaraConan', 'jsee23/public',
'foxostro/PinkTopaz', 'jgsogo/conan-packages',
'vtpl1/conan-expat', 'vtpl1/conan-ffmpeg', 'ess-dmsc/conan',
'acdemiralp/makina', 'rwth-vr/conan', 'onyx/conan'
]:
splitted = a.split('/')
repos.add(BintrayRepoDescriptor(splitted[0], splitted[1]))
return repos
def _find_packages(
self,
repos: Set[BintrayRepoDescriptor]) -> [BintrayPackageDescriptor]:
"""
Returns all packages in the known repositories
"""
pkgs = [
BintrayPackageDescriptor(repo.repoowner, repo.reponame, p['name'],
p['linked']) for repo in repos
for p in self.client.get_all('/repos/' + repo.repoowner + '/' +
repo.reponame + '/packages')
]
# replace linked packages by their sources
for index, pkg in enumerate(pkgs):
if pkg.linked:
bt_package = self.client.get('/packages/' + pkg.repoowner +
'/' + pkg.reponame + '/' +
pkg.name)
pkgs[index] = BintrayPackageDescriptor(bt_package['owner'],
bt_package['repo'],
pkg.name, False)
return list(set(pkgs))
@classmethod
def _default_package_filename(cls, pkg: BintrayPackageDescriptor):
clean_name = pkg.name.split(':')[0].lower().replace('conan-', '')
clean_name = re.sub(r'^boost_', 'boost.', clean_name)
clean_name = clean_name.replace('+', '_')
return os.path.join(packages_directory(), clean_name + '.yaml')
def _filter_missing_package(
self, packages: [BintrayPackageDescriptor
]) -> [BintrayPackageDescriptor]:
"""Only return packages for those there is no package file available currently"""
return [
p for p in packages
if not os.path.exists(self._default_package_filename(p))
]
def _filter_missing_repository(
self, packages: [BintrayPackageDescriptor
]) -> [BintrayPackageDescriptor]:
"""Only return packages that have missing repository descriptors"""
return [
p for p in packages if '/'.join(p[:3]) not in self.existing_repos
and not p.name.startswith('Boost') and p.reponame != 'conan-center'
]
def run(self):
found = self._find_packages(self._gather_repos())
self.missing_packages = self._filter_missing_package(found)
self.missing_repos = sorted(self._filter_missing_repository(found))
def print(self):
for pkg in self.missing_packages:
print('Missing package:', '/'.join(pkg[:3]))
for pkg in self.missing_repos:
print('Missing repository:', '/'.join(pkg[:3]))
def generate_stubs(self):
for name in self.missing_packages:
# pkg = self.http.get(self.url + '/' + name).json()
fname = self._default_package_filename(name)
logging.getLogger(__name__).info('Generating %s',
os.path.basename(fname))
with open(fname, 'w') as f:
yaml.dump(
dict(recipies=[
dict(repo=dict(bintray=name.repoowner + '/' +
name.reponame + '/' + name.name))
],
urls=dict()), f)