def check_services_on_branch(versions_block):
"""
Returns the list of all services that are on a branch.
"""
services_on_branch = []
for service in versions_block:
version = versions_block.get(service)
if "quay.io/cdis" not in version or len(version.split(":")) < 2:
# ignore non-CTDS repos.
# version without ":" is not usable.
continue
version = version.split(":")[1]
if version_is_branch(version, release_tag_are_branches=False):
services_on_branch.append(service)
return services_on_branch
async def generate(hub, **pkginfo):
python_compat = "python3+"
user = "******"
name = pkginfo["name"]
releases_data = await hub.pkgtools.fetch.get_page(
f"https://api.github.com/repos/{user}/{name}/releases", is_json=True)
latest_release = get_release(releases_data)
if latest_release is None:
raise hub.pkgtools.ebuild.BreezyError(
f"Can't find a suitable release of {name}")
version = latest_release["tag_name"]
major_version = version.split(".", 1)[0]
ebuild = hub.pkgtools.ebuild.BreezyBuild(
**pkginfo,
version=version,
major_version=major_version,
python_compat=python_compat,
artifacts=[
hub.pkgtools.ebuild.Artifact(
url=
f"https://github.com/{user}/{name}/archive/{version}.tar.gz",
final_name=f"{name}-{version}.tar.gz",
)
],
)
ebuild.push()
def _ensure_min_version(cmake, dry_run):
cmd = [cmake, '--version']
if dry_run:
log.inf('Dry run:', quote_sh_list(cmd))
return
try:
version_out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as cpe:
log.die('cannot get cmake version:', str(cpe))
decoded = version_out.decode('utf-8')
lines = decoded.splitlines()
if not lines:
log.die('can\'t get cmake version: ' +
'unexpected "cmake --version" output:\n{}\n'.format(decoded) +
'Please install CMake ' + _MIN_CMAKE_VERSION_STR +
' or higher (https://cmake.org/download/).')
version = lines[0].split()[2]
if '-' in version:
# Handle semver cases like "3.19.20210206-g1e50ab6"
# which Kitware uses for prerelease versions.
version = version.split('-', 1)[0]
if packaging.version.parse(version) < _MIN_CMAKE_VERSION:
log.die(
'cmake version', version,
'is less than minimum version {};'.format(_MIN_CMAKE_VERSION_STR),
'please update your CMake (https://cmake.org/download/).')
else:
log.dbg('cmake version', version, 'is OK; minimum version is',
_MIN_CMAKE_VERSION_STR)
def parse(cls, software: str, version: str) -> 'ServerInfo':
if software == "Synapse":
return ServerInfo(software=software,
version=packaging.version.parse(
version.split(" ")[0]))
else:
return ServerInfo(software=software, version=version)
def parse(cls, software: str, version: str) -> 'ServerInfo':
if software.lower() == "synapse":
return ServerInfo(software="Synapse",
version=packaging.version.parse(
version.split(" ")[0]))
elif software.lower() == "dendrite":
return ServerInfo(software="Dendrite",
version=semver.VersionInfo.parse(version))
else:
return ServerInfo(software=software, version=version)
def setup(version, verbose=True):
s3_url = ''
type_n_version = version.split(':', 1)
if len(type_n_version) == 2:
s3_version = type_n_version[1]
s3_url = RELOCATABLE_URLS_BASE.format(type_n_version[0], s3_version)
version = os.path.join(*type_n_version)
cdir = version_directory(version)
if cdir is None:
tmp_download = tempfile.mkdtemp()
url = os.environ.get('SCYLLA_CORE_PACKAGE',
os.path.join(s3_url, 'scylla-package.tar.gz'))
package_version = download_version(version,
verbose=verbose,
url=url,
target_dir=os.path.join(
tmp_download,
'scylla-core-package'))
# Try the old name for backward compatibility
url = os.environ.get("SCYLLA_TOOLS_JAVA_PACKAGE") or \
os.environ.get("SCYLLA_JAVA_TOOLS_PACKAGE") or \
os.path.join(s3_url, 'scylla-tools-package.tar.gz')
download_version(version,
verbose=verbose,
url=url,
target_dir=os.path.join(tmp_download,
'scylla-tools-java'))
url = os.environ.get('SCYLLA_JMX_PACKAGE',
os.path.join(s3_url, 'scylla-jmx-package.tar.gz'))
download_version(version,
verbose=verbose,
url=url,
target_dir=os.path.join(tmp_download, 'scylla-jmx'))
cdir = directory_name(version)
shutil.move(tmp_download, cdir)
# install using scylla install.sh
run_scylla_install_script(os.path.join(cdir, 'scylla-core-package'),
cdir, package_version)
setup_scylla_manager()
return cdir, get_scylla_version(cdir)
def pupil_version_string() -> str:
"""
[major].[minor].[trailing-untagged-commits]
"""
# NOTE: This returns the current version as read from the last git tag. Normally you
# don't want to use this, but get_version() below, which also works in a bundled
# version (without git).
version = get_tag_commit()
if version is None:
raise ValueError("Version Error")
try:
parts_git_tag = version.split("-")
version_parsed = packaging.version.Version(parts_git_tag[0])
if version_parsed.is_prerelease:
version = version_parsed.base_version
except packaging.version.InvalidVersion:
pass
version = version.replace("v", "") # strip version 'v'
# print(version)
if "-" in version:
parts = version.split("-")
version = ".".join(parts[:-1])
return version
def get_splitted(version: str) -> Tuple[List[str], str]:
"""
Splits the version from its code name.
:param version:
The version to split.
:return:
A tuple. The first index is the digit part of the version,
when the second one is the the non-digit part of the
version.
As example if :code:`1.0.0. Hello World` is given, this method
will output:
::
([1,0,0], " Hello World")
"""
splitted_version: List[str] = version.split(".")
def get_version_part() -> List[str]:
"""
Provides the version part.
"""
return [
x for x in splitted_version if x.isdigit() or x[0].isdigit()
]
def get_codename_part() -> str:
"""
Provides the codename part.
"""
try:
return [
x for x in splitted_version
if not x.isdigit() and not x[0].isdigit()
][0]
except IndexError:
return ""
return get_version_part(), get_codename_part()
def __init__(self, version):
"""
Initialize a new Version.
Parameters:
version: str, Version
The semver-formatted version string; or another Version instance.
"""
if isinstance(version, Version):
version = str(version)
if not isinstance(version, str):
raise TypeError("version")
#support versions such as: "application/vnd.mds.provider+json;version=0.3"
if version.startswith('application/vnd.mds.provider+json;version='):
version = version.split('=')[1]
self._version = self._parse(version)
self._legacy = None
if isinstance(self._version, packaging.version.LegacyVersion):
# versions like "0.3.x" or "0.x"
try:
# assume the highest PATCH support
major, minor, legacy = str(self._version).split(".")
self._version = self._parse(f"{major}.{minor}.{sys.maxsize}")
# note the highest valid version tuple index, and the "legacy" data
self._legacy = (1, legacy)
except:
# assume the highest MINOR.PATCH support
major, legacy = str(self._version).split(".")
self._version = self._parse(
f"{major}.{sys.maxsize}.{sys.maxsize}")
# note the highest valid version tuple index, and the "legacy" data
self._legacy = (0, legacy)
elif len(self.tuple) < 2:
# MAJOR only versions like "0", "1"
self._version = self._parse(
f"{self.tuple[0]}.{sys.maxsize}.{sys.maxsize}")
self._legacy = (0, None)
elif len(self.tuple) < 3:
# MAJOR.MINOR only version like "0.3"
self._version = self._parse(
f"{self.tuple[0]}.{self.tuple[1]}.{sys.maxsize}")
self._legacy = (1, None)
def format_version(version: str, fmt: str = fmt_dev, is_dev: bool = False):
branch = get_branch()
if branch is not None and branch.startswith("release-"):
_, version = branch.split("release-")
return version
parts = version.split('-')
assert len(parts) in (3, 4)
dirty = len(parts) == 4
tag, count, sha = parts[:3]
MA, MI, PA = map(int, tag.split(".")) # 0.9.0 -> (0, 9, 0)
if is_dev:
tag = f"{MA}.{MI + 1}.{0}"
if count == '0' and not dirty:
return tag
return fmt.format(tag=tag, gitsha=sha.lstrip('g'))
def increment_version(version):
'''
:param version: the semantic versioning string, increments the patch number
by one
:type version: str
'''
version_lst = version.split('.')
patch_version = version_lst[2]
if patch_version.isdigit():
patch_version = int(patch_version) + 1
version_lst[2] = str(patch_version)
else:
msg = 'The patch version cannot be incremented as it is not a number. ' + \
'version is: {0}, and extracted patch number is {1}'
msg = msg.format(version, patch_version)
raise ValueError(msg)
newversion = '.'.join(version_lst)
return newversion
def version_to_code(cls, version, flavor="stable"):
"""Convert version into corresponding code.
Args:
version (str): Version string
flavor (str, optional): Android flavor
Returns:
int: Version code
"""
if flavor == "stable":
# Convert 'x.y.z' to 'XXXYYYZZZ'
major, minor, bug = version.split(".")
code_str = "{0:0>3}{1:0>3}{2:0>3}".format(major, minor, bug)
# Get rid of leading zeros.
code = int(code_str)
else:
raise NotImplementedError
return code
def _get_platform_tags():
# type: () -> Sequence[str]
if sys.platform == "darwin":
version, _, arch = platform.mac_ver()
major, minor_str = version.split(".")[:2]
tags = []
minor = int(minor_str)
while minor >= 6:
tags.append("macosx_{major}_{minor}_{arch}".format(major=major,
minor=minor,
arch=arch))
tags.append("macosx_{major}_{minor}_intel".format(major=major,
minor=minor))
minor -= 1
tag = tuple(tags)
else:
plat = distutils.util.get_platform() # pylint: disable=no-member
tag = (plat.replace(".", "_").replace("-", "_"), )
return ("any", ) + tag
def parse_version(version):
# type: (str) -> typing.Tuple[int, int, int]
"""Convert a version string to a tuple of (major, minor, micro)
Examples::
1.2.3 -> (1, 2, 3)
1.2 -> (1, 2, 0)
1 -> (1, 0, 0)
1.0.0-beta1 -> (1, 0, 0)
2020.6.19 -> (2020, 6, 19)
malformed -> (0, 0, 0)
10.5.0 extra -> (10, 5, 0)
"""
# If we have any spaces/extra text, grab the first part
# "1.0.0 beta1" -> "1.0.0"
# "1.0.0" -> "1.0.0"
# DEV: Versions with spaces will get converted to LegacyVersion, we do this splitting
# to maximize the chances of getting a Version as a parsing result
if " " in version:
version = version.split()[0]
# version() will not raise an exception, if the version if malformed instead
# we will end up with a LegacyVersion
parsed = packaging.version.parse(version)
# LegacyVersion.release will always be `None`
if not parsed.release:
return (0, 0, 0)
# Version.release was added in 17.1
# packaging >= 20.0 has `Version.{major,minor,micro}`, use the following
# to support older versions of the library
# https://github.com/pypa/packaging/blob/47d40f640fddb7c97b01315419b6a1421d2dedbb/packaging/version.py#L404-L417
return (
parsed.release[0] if len(parsed.release) >= 1 else 0,
parsed.release[1] if len(parsed.release) >= 2 else 0,
parsed.release[2] if len(parsed.release) >= 3 else 0,
)
def get_release_data(repo_name, git_api, is_add_version=False):
"""Get release data from Github"""
repo = git_api.get_repo(repo_name)
releases = repo.get_releases()
gem_name = os.path.basename(repo_name)
gem_data_list = []
for rel in releases:
if rel.prerelease: # ignore pre releases
continue
version = rel.tag_name.lstrip('v')
li = version.split('-')
if len(li) > 1: # ignore alpha or beta releases
continue
gem_data = {}
rel_id = f"{gem_name}-{version}"
gem_data['id'] = rel_id
if rel_id in EXT_PARENT_ID:
gem_data['externalParentId'] = EXT_PARENT_ID[rel_id]
else:
gem_data['externalParentId'] = []
gem_data['releaseDate'] = rel.published_at.strftime("%Y-%m-%d")
if is_add_version:
gem_data['version'] = version
gem_data['releaseLink'] = rel.html_url
if rel_id in RELEASE_PMID_DICT:
gem_data['PMID'] = RELEASE_PMID_DICT[rel_id]
elif gem_name in GEM_PMID_DICT:
gem_data['PMID'] = GEM_PMID_DICT[gem_name]
else:
gem_data['PMID'] = ''
gem_data['count'] = get_count(repo_name, git_api)
gem_data['validation'] = get_validation(repo_name, git_api)
gem_data_list.append(gem_data)
# sort the releases in the ascending order
gem_data_list = gem_data_list[::-1]
return gem_data_list
def _get_specs(specset):
if specset is None:
return
if is_instance(specset, Specifier):
new_specset = SpecifierSet()
specs = set()
specs.add(specset)
new_specset._specs = frozenset(specs)
specset = new_specset
if isinstance(specset, str):
specset = SpecifierSet(specset)
result = []
for spec in set(specset):
version = spec.version
op = spec.operator
if op in ("in", "not in"):
versions = version.split(",")
op = "==" if op == "in" else "!="
for ver in versions:
result.append((op, _tuplize_version(ver.strip())))
else:
result.append((spec.operator, _tuplize_version(spec.version)))
return sorted(result, key=operator.itemgetter(1))
def _tuplize_version(version):
return tuple(int(x) for x in filter(lambda i: i != "*", version.split(".")))
def _tuplize_version(version):
# type: (STRING_TYPE) -> Tuple[int, ...]
return tuple(int(x) for x in filter(lambda i: i != "*", version.split(".")))
def _parse(version):
# type: (str) -> Tuple[int, ...]
if isinstance(version, STRING_TYPES):
return tuple(int(i) for i in version.split("."))
return version
class LibStatismoConan(ConanFile):
name = "statismo"
version = "0.12.1"
version_split = version.split('.')
description = "Framework for building Statistical Image And Shape Models"
url = "https://github.com/kenavolic/statismo"
license = "BSD-3-clause"
exports_sources = "*"
generators = "cmake", "cmake_paths", "cmake_find_package_multi"
settings = "os", "arch", "compiler", "build_type"
options = {
"shared": [True, False],
"fPIC": [True, False],
"with_tests": [True, False],
"with_logs": [True, False]
# TODO: support more options as required by client projects
}
default_options = {
"shared": True,
"fPIC": True,
"with_tests": True,
"with_logs": False
# TODO: support more options as required by client projects
}
_build_subfolder = "build_subfolder"
_cmake = None
def config_options(self):
if self.settings.compiler == "Visual Studio":
del self.options.fPIC
def configure(self):
if self.settings.compiler == 'gcc' and self.settings.compiler.libcxx != 'libstdc++11':
raise ConanInvalidConfiguration(
"Statismo requires ABI 'libstdc++11' when built with 'gcc'")
tools.check_min_cppstd(self, "14")
self.options["hdf5"].shared = self.options.shared
def source(self):
# No-op for now
pass
def requirements(self):
self.requires("cmake/3.16.3")
self.requires("eigen/3.3.7")
self.requires("hdf5/1.10.6")
self.requires("itk/5.0.1@user/stable")
self.requires("vtk/8.2.0@user/stable")
def _configure_cmake(self):
if self._cmake is not None:
return self._cmake
self._cmake = CMake(self)
# Add this to include package root as module paths (needed for proper VTK/ITK inclusion)
self._cmake.definitions["CMAKE_TOOLCHAIN_FILE"] = "conan_paths.cmake"
# Build only necessary modules
self._cmake.definitions["BUILD_DOCUMENTATION"] = "OFF"
self._cmake.definitions["BUILD_TESTS"] = self.options.with_tests
self._cmake.definitions["BUILD_TESTING"] = self.options.with_tests
self._cmake.definitions["BUILD_EXAMPLES"] = "ON"
self._cmake.definitions["USE_ITK_EIGEN"] = "OFF"
self._cmake.definitions["USE_ITK_HDF5"] = "OFF"
self._cmake.definitions["BUILD_SHARED_LIBS"] = self.options.shared
self._cmake.definitions["ENABLE_RUNTIME_LOGS"] = self.options.with_logs
# TODO: Update when more options will be added
if self.settings.os == "Macos":
self.env["DYLD_LIBRARY_PATH"] = os.path.join(
self.build_folder, "lib")
self.output.info("cmake build: %s" % self.build_folder)
self._cmake.configure(build_folder=self._build_subfolder)
return self._cmake
def build(self):
cmake = self._configure_cmake()
environ_backup = os.environ.copy()
os.environ['CONAN_CPU_COUNT'] = '2'
try:
if self.settings.os == "Macos":
# run_environment does not work here because it appends path just from
# requirements, not from this package itself
# https://docs.conan.io/en/latest/reference/build_helpers/run_environment.html#runenvironment
lib_path = os.path.join(self.build_folder, "lib")
self.run("DYLD_LIBRARY_PATH={0} cmake --build . {1}".format(
lib_path, cmake.build_config))
else:
cmake.build()
finally:
os.environ.clear()
os.environ.update(environ_backup)
if self.options.with_tests:
env_build = RunEnvironment(self)
with tools.environment_append(env_build.vars):
cmake.parallel = False
cmake.verbose = True
cmake.test(output_on_failure=True)
def package(self):
cmake = self._configure_cmake()
cmake.install()
cmake.patch_config_paths()
def package_info(self):
self.cpp_info.names["cmake_find_package"] = "statismo"
self.cpp_info.names["cmake_find_package_multi"] = "statismo"
self.cpp_info.libs = tools.collect_libs(self)
self.cpp_info.includedirs = ["include"]
def setup(version, verbose=True):
"""
:param version:
Supported version values (examples):
1. Unstable versions:
- unstable/master:2020-12-20T00:11:59Z
- unstable/enterprise:2020-08-18T14:49:18Z
- unstable/branch-4.1:2020-05-30T08:27:59Z
2. Official version (released):
- release:4.3
- release:4.2.1
- release:2020.1 (SCYLLA_PRODUCT='enterprise')
- release:2020.1.5 (SCYLLA_PRODUCT='enterprise')
"""
s3_url = ''
type_n_version = version.split(':', 1)
scylla_product = os.environ.get('SCYLLA_PRODUCT', 'scylla')
packages = None
if len(type_n_version) == 2:
s3_version = type_n_version[1]
packages = None
if type_n_version[0] == 'release':
if 'enterprise' in scylla_product:
s3_url = ENTERPRISE_RELEASE_RELOCATABLE_URLS_BASE
else:
s3_url = RELEASE_RELOCATABLE_URLS_BASE
packages, type_n_version[1] = release_packages(s3_url=s3_url, version=s3_version)
else:
_, branch = type_n_version[0].split("/")
if 'enterprise' in scylla_product:
s3_url = get_relocatable_s3_url(branch, s3_version, ENTERPRISE_RELOCATABLE_URLS_BASE)
else:
s3_url = get_relocatable_s3_url(branch, s3_version, RELOCATABLE_URLS_BASE)
packages = RelocatablePackages(scylla_jmx_package=os.path.join(s3_url,
f'{scylla_product}-jmx-package.tar.gz'),
scylla_tools_package=os.path.join(s3_url,
f'{scylla_product}-tools-package.tar.gz'),
scylla_package=os.path.join(s3_url,
f'{scylla_product}-package.tar.gz'))
version = os.path.join(*type_n_version)
version_dir = version_directory(version)
if version_dir is None:
# Create version folder and add placeholder file to prevent parallel downloading from another test.
version_dir = directory_name(version)
download_in_progress_file = Path(version_dir) / DOWNLOAD_IN_PROGRESS_FILE
# Give a chance not to start few downloads in the exactly same second
time.sleep(random.randint(0, 5))
# If another parallel downloading has been started already, wait while it will be completed
if download_in_progress_file.exists():
print(f"Another download running into '{version_dir}'. Waiting for parallel downloading finished")
wait_for_parallel_download_finish(placeholder_file=download_in_progress_file.absolute())
else:
try:
os.makedirs(version_dir)
except FileExistsError as exc:
# If parallel process created the folder first, let to the parallel download to finish
print(f"Another download running into '{version_dir}'. Waiting for parallel downloading finished")
wait_for_parallel_download_finish(placeholder_file=download_in_progress_file.absolute())
else:
download_in_progress_file.touch()
try:
package_version = download_packages(version_dir=version_dir, packages=packages, s3_url=s3_url,
scylla_product=scylla_product, version=version, verbose=verbose)
except requests.HTTPError as err:
if '404' in err.args[0]:
packages_x86_64 = RelocatablePackages(
scylla_jmx_package=os.path.join(s3_url, f'{scylla_product}-jmx-package.tar.gz'),
scylla_tools_package=os.path.join(s3_url, f'{scylla_product}-tools-package.tar.gz'),
scylla_package=os.path.join(s3_url, f'{scylla_product}-x86_64-package.tar.gz')
)
package_version = download_packages(version_dir=version_dir, packages=packages_x86_64,
s3_url=s3_url, scylla_product=scylla_product,
version=version, verbose=verbose)
else:
raise
download_in_progress_file.touch()
# install using scylla install.sh
run_scylla_install_script(install_dir=os.path.join(version_dir, CORE_PACKAGE_DIR_NAME),
target_dir=version_dir,
package_version=package_version)
print(f"Completed to install Scylla in the folder '{version_dir}'")
download_in_progress_file.unlink()
scylla_ext_opts = os.environ.get('SCYLLA_EXT_OPTS', '')
scylla_manager_package = os.environ.get('SCYLLA_MANAGER_PACKAGE')
if scylla_manager_package:
manager_install_dir = setup_scylla_manager(scylla_manager_package)
scylla_ext_opts += ' --scylla-manager={}'.format(manager_install_dir)
os.environ['SCYLLA_EXT_OPTS'] = scylla_ext_opts
return version_dir, get_scylla_version(version_dir)
def release_packages(s3_url, version):
"""
Choose RELEASE relocatable packages for download.
It covers 3 cases:
1. Choose release packages for supplied version. If scylla_version includes minor version (like "4.3.1"),
relocatable packages for this version will be selected.
2. Choose release packages for latest release version. If scylla_version doesn't include minor version (like "4.3),
relocatables for latest released version will be selected.
3. If the version was not released yet, relocatables for latest release candidate (rc0..) will be selected.
:param s3_url:
:param version:
:return:
"""
major_version = version if version.count('.') == 1 else '.'.join(n for n in version.split('.')[:2])
s3_url = s3_url.format(major_version)
files_in_bucket = aws_bucket_ls(s3_url)
if not files_in_bucket:
raise RuntimeError(
f"Failed to get release packages list for version {major_version}. URL: {s3_url}.")
all_packages = [name for name in files_in_bucket if 'jmx' in name or 'tools' in name or 'scylla-package' in name]
candidates = []
# Search for released version packages
if '.rc' not in version:
candidates = [package for package in all_packages if ".rc" not in package]
if candidates:
# If found packages for released version
if version.count('.') == 1:
# Choose relocatable packages for latest release version
a = re.findall(f'-{version}\.(\d+)(?:-|\.)', ','.join(candidates))
latest_candidate = f"{version}.{max(set(a))}" if a else ''
elif version.count('.') == 2:
# Choose relocatable packages for supplied version
latest_candidate = version
else:
raise ValueError(f"Not expected version number: {version}. S3_URL: {s3_url}")
release_packages = [package for package in candidates if f"-{latest_candidate}" in package]
else:
# Choose relocatables for latest release candidate
if '.rc' in version:
latest_candidate = version
else:
a = re.findall('\.rc(\d+)(?:-|\.)', ','.join(all_packages))
latest_candidate = f"{version}.rc{max(set(a))}" if a else ""
release_packages = [package for package in all_packages if latest_candidate in package]
if not release_packages:
raise ValueError(
f"Release packages have not beed found.\nDebug info: all packages: {all_packages}; "
f"candidates packages: {candidates}; last version: {latest_candidate}")
release_packages_dict = {}
for package in release_packages:
# Expected packages names (examples):
# 'scylla-jmx-package-4.3.0-0.20210110.000585522.tar.gz'
# 'scylla-package-4.3.0-0.20210110.000585522.tar.gz'
# 'scylla-tools-package-4.3.0-0.20210110.000585522.tar.gz'
release_packages_dict[package.split("-")[1]] = package
packages = RelocatablePackages(scylla_package=os.path.join(s3_url, release_packages_dict['package']),
scylla_jmx_package=os.path.join(s3_url, release_packages_dict['jmx']),
scylla_tools_package=os.path.join(s3_url, release_packages_dict['tools']))
return packages, latest_candidate
