def parse(version_str):
'''Compatibility function to normalize version strings from prior
ReFrame versions
:returns: a :class:`semver.VersionInfo` object.
'''
compat = False
old_style_stable = re.search(r'^(\d+)\.(\d+)$', version_str)
old_style_dev = re.search(r'(\d+)\.(\d+)((\d+))?-dev(\d+)$', version_str)
if old_style_stable:
compat = True
major = old_style_stable.group(1)
minor = old_style_stable.group(2)
ret = semver.VersionInfo(major, minor, 0)
elif old_style_dev:
compat = True
major = old_style_dev.group(1)
minor = old_style_dev.group(2)
patchlevel = old_style_dev.group(4) or 0
prerelease = old_style_dev.group(5)
ret = semver.VersionInfo(major, minor, patchlevel, f'dev.{prerelease}')
else:
ret = semver.VersionInfo.parse(version_str)
if compat:
user_deprecation_warning(
f"the version string {version_str!r} is deprecated; "
f"please use the conformant '{ret}'",
from_version='3.5.0')
return ret
def _parse_version(cls, version):
# Consider a 4th digit to be a SemVer pre-release.
# E.g. 1.2.3.4 is 1.2.3-4
m = cls.FOUR_DIGIT_VERSION_REGEX.match(version)
if m:
major, minor, patch, prerelease = m.groups()
return semver.VersionInfo(major=major,
minor=minor,
patch=patch,
prerelease=prerelease)
# Consider a "dev" build to be a SemVer pre-release.
# E.g. 0.0.209dev12 is 0.0.209-12
m = cls.DEV_VERSION_REGEX.match(version)
if m:
major, minor, patch, prerelease = m.groups()
return semver.VersionInfo(major=major,
minor=minor,
patch=patch,
prerelease=prerelease)
# Otherwise hope that the semver package can deal with it.
try:
return semver.VersionInfo.parse(version)
except ValueError as e:
lib.log_error(str(e))
# At least return something comparable.
return semver.VersionInfo(major=0)
def setUp(self):
builds = {
ImageBuild(semver.VersionInfo(1, 0, 0, None, None), ('option',)),
ImageBuild(semver.VersionInfo(1, 0, 0, None, None), (None,)),
}
self.matrix = BuildMatrix(builds)
def btc_sign_msg(
self, coin: btc.BTCCoin, script_config: btc.BTCScriptConfigWithKeypath, msg: bytes
) -> Tuple[bytes, int, bytes]:
"""
Returns a 64 byte sig, the recoverable id, and a 65 byte signature containing
the recid, compatible with Electrum.
"""
# pylint: disable=no-member,line-too-long
self._require_atleast(semver.VersionInfo(9, 2, 0))
request = btc.BTCRequest()
request.sign_message.CopyFrom(
btc.BTCSignMessageRequest(coin=coin, script_config=script_config, msg=msg)
)
supports_antiklepto = self.version >= semver.VersionInfo(9, 5, 0)
if supports_antiklepto:
host_nonce = os.urandom(32)
request.sign_message.host_nonce_commitment.commitment = antiklepto_host_commit(
host_nonce
)
signer_commitment = self._btc_msg_query(
request, expected_response="antiklepto_signer_commitment"
).antiklepto_signer_commitment.commitment
request = btc.BTCRequest()
request.antiklepto_signature.CopyFrom(
antiklepto.AntiKleptoSignatureRequest(host_nonce=host_nonce)
)
signature = self._btc_msg_query(
request, expected_response="sign_message"
).sign_message.signature
antiklepto_verify(host_nonce, signer_commitment, signature[:64])
if self.debug:
print(f"Antiklepto nonce verification PASSED")
else:
signature = self._btc_msg_query(
request, expected_response="sign_message"
).sign_message.signature
sig, recid = signature[:64], signature[64]
# See https://github.com/spesmilo/electrum/blob/84dc181b6e7bb20e88ef6b98fb8925c5f645a765/electrum/ecc.py#L521-L523
compressed = 4 # BitBox02 uses only compressed pubkeys
electrum_sig65 = bytes([27 + compressed + recid]) + sig
return (sig, recid, electrum_sig65)
def test_detects_the_latest_versions(self):
builds = {
ImageBuild(semver.VersionInfo(1, 0, 0, None, None), ()),
ImageBuild(semver.VersionInfo(1, 0, 1, None, None), ()),
ImageBuild(semver.VersionInfo(1, 1, 0, None, None), ()),
ImageBuild(semver.VersionInfo(2, 0, 0, None, None), ()),
}
self.matrix = BuildMatrix(builds)
self.assertDictEqual(
{'1.0.1': {'1.0'}, '1.1.0': {'1', '1.1'}, '2.0.0': {'2', '2.0'}},
self.matrix.latest
)
def coerce(version):
"""
Convert an incomplete version string into a semver-compatible VersionInfo
object
* Tries to detect a "basic" version string (``major.minor.patch``).
* If not enough components can be found, missing components are
set to zero to obtain a valid semver version.
:param str version: the version string to convert
:return: a tuple with a :class:`VersionInfo` instance (or ``None``
if it's not a version) and the rest of the string which doesn't
belong to a basic version.
:rtype: tuple(:class:`VersionInfo` | None, str)
"""
match = BASEVERSION.search(version)
if not match:
return (None, version)
ver = {
key: 0 if value is None else value for key, value in match.groupdict().items()
}
ver = semver.VersionInfo(**ver)
rest = match.string[match.end() :] # noqa:E203
return ver, rest
def tag(ctx, tag=None, major=False, minor=False, patch=False):
"""Tag or bump release with a semver tag, prefixed with 'v'. Makes a signed tag."""
latest = None
if tag is None:
tags = get_tags()
if not tags:
latest = semver.VersionInfo(0, 0, 0)
else:
latest = tags[-1]
if patch:
nextver = latest.bump_patch()
elif minor:
nextver = latest.bump_minor()
elif major:
nextver = latest.bump_major()
else:
nextver = latest.bump_patch()
else:
if tag.startswith("v"):
tag = tag[1:]
try:
nextver = semver.parse_version_info(tag)
except ValueError:
raise Exit("Invalid semver tag.", 2)
print(latest, "->", nextver)
tagopt = "-s" if CURRENT_USER in SIGNERS else "-a"
ctx.run(f'git tag {tagopt} -m "Release v{nextver}" v{nextver}')
def user_deprecation_warning(message, from_version='0.0.0'):
'''Raise a deprecation warning at the user stack frame that eventually
calls this function.
As "user stack frame" is considered a stack frame that is outside the
:py:mod:`reframe` base module.
:arg message: the message of the warning
:arg from_version: raise the warning only for ReFrame versions greater than
this one. This is useful if you want to "schedule" a deprecation
warning for the future
'''
# Unwind the stack and issue the warning from the first stack frame that is
# outside the framework.
stack_level = 1
for s in inspect.stack():
module = inspect.getmodule(s.frame)
if module is None or not module.__name__.startswith('reframe'):
break
stack_level += 1
min_version = semver.VersionInfo.parse(from_version)
version = semver.VersionInfo.parse(reframe.VERSION)
if version.prerelease:
# Promote prereleases, so that we issue the warning also in this case
version = semver.VersionInfo(version.major, version.minor,
version.patch)
if _RAISE_DEPRECATION_ALWAYS or version >= min_version:
warnings.warn(message,
ReframeDeprecationWarning,
stacklevel=stack_level)
def nexus_mock_client(mocker, faker):
"""A nexus_client with the request method mocked"""
class ResponseMock:
def __init__(self):
self.status_code = 200
self.content = faker.sentence()
self.reason = faker.sentence()
# prepare content for repositories.refresh()
self._json = [
nexus_repository(name=faker.pystr(),
format_=faker.random.choice([
'pypi', 'nuget', 'raw', 'yum', 'rubygems'
])) for _ in range(faker.random_int(1, 10))
]
def json(self):
return self._json
mocker.patch('nexuscli.nexus_client.NexusClient.http_request',
return_value=ResponseMock())
client = NexusClient()
client._server_version = semver.VersionInfo(3, 19, 0)
client.repositories.refresh()
return client
def btc_sign_msg(self, coin: btc.BTCCoin,
script_config: btc.BTCScriptConfigWithKeypath,
msg: bytes) -> Tuple[bytes, int, bytes]:
"""
Returns a 64 byte sig, the recoverable id, and a 65 byte signature containing
the recid, compatible with Electrum.
"""
# pylint: disable=no-member,line-too-long
self._require_atleast(semver.VersionInfo(9, 2, 0))
request = btc.BTCRequest()
request.sign_message.CopyFrom(
btc.BTCSignMessageRequest(coin=coin,
script_config=script_config,
msg=msg))
sig = self._btc_msg_query(
request, expected_response="sign_message").sign_message.signature
sig, recid = sig[:64], sig[64]
# See https://github.com/spesmilo/electrum/blob/84dc181b6e7bb20e88ef6b98fb8925c5f645a765/electrum/ecc.py#L521-L523
compressed = 4 # BitBox02 uses only compressed pubkeys
electrum_sig65 = bytes([27 + compressed + recid]) + sig
return (sig, recid, electrum_sig65)
def test_check_version(vibium):
# Get current API version for testing
session = MVG(vibium, "NO TOKEN")
# Check OK run
session.tested_api_version = session.api_version
assert session.check_version()["api_version"] == session.api_version
# Check Incompatible Major version
session.tested_api_version = semver.VersionInfo(major=100)
with pytest.raises(ValueError):
session.check_version()
# Check to high highest tested version
session.tested_api_version = session.api_version.bump_minor()
with pytest.raises(ValueError):
session.check_version()
# Check API minor version higher
session.api_version = session.tested_api_version.bump_minor()
with pytest.raises(UserWarning):
session.check_version()
# Check version with pre-release
session.tested_api_version = session.api_version.bump_prerelease()
session.check_version()
def eth_sign_msg(
self, msg: bytes, keypath: Sequence[int], coin: eth.ETHCoin = eth.ETH
) -> bytes:
"""
Signs message, the msg will be prefixed with "\x19Ethereum message\n" + len(msg) in the
hardware
"""
request = eth.ETHRequest()
# pylint: disable=no-member
request.sign_msg.CopyFrom(eth.ETHSignMessageRequest(coin=coin, keypath=keypath, msg=msg))
supports_antiklepto = self.version >= semver.VersionInfo(9, 5, 0)
if supports_antiklepto:
host_nonce = os.urandom(32)
request.sign_msg.host_nonce_commitment.commitment = antiklepto_host_commit(host_nonce)
signer_commitment = self._eth_msg_query(
request, expected_response="antiklepto_signer_commitment"
).antiklepto_signer_commitment.commitment
request = eth.ETHRequest()
request.antiklepto_signature.CopyFrom(
antiklepto.AntiKleptoSignatureRequest(host_nonce=host_nonce)
)
signature = self._eth_msg_query(request, expected_response="sign").sign.signature
antiklepto_verify(host_nonce, signer_commitment, signature[:64])
if self.debug:
print(f"Antiklepto nonce verification PASSED")
return signature
return self._eth_msg_query(request, expected_response="sign").sign.signature
def get_current_version(stage: str = None) -> str:
"check the latest release from github"
stage = stage if stage else args.stage
version_url = f'https://api.github.com/repos/{repo}/releases'
releases = requests.get(version_url).json()
# would use version['target_commitish'] to grab the stage, but in use it grabs unexpected stages
if releases and stage == 'integration':
versions = [
semver.parse_version_info(version['tag_name'])
for version in releases
if semver.parse_version_info(version['tag_name']).prerelease
and semver.parse_version_info(
version['tag_name']).prerelease.startswith('integration')
]
elif releases and stage == 'staging':
versions = [semver.parse_version_info(version['tag_name']) for version in releases
if semver.parse_version_info(version['tag_name']).prerelease
and semver.parse_version_info(version['tag_name']).prerelease.startswith('rc')] \
or get_current_version('integration')
elif releases and stage == 'prod':
versions = [
semver.parse_version_info(version['tag_name'])
for version in releases
if not semver.parse_version_info(version['tag_name']).prerelease
]
if not versions:
versions = [semver.VersionInfo(0, 0, 0)]
return str(max(versions))
def device_info(self) -> Dict[str, Any]:
"""
Returns an object with device information, e.g. name, passphrase status, etc.
"""
# pylint: disable=no-member
request = hww.Request()
device_info_request = bitbox02_system.DeviceInfoRequest()
request.device_info.CopyFrom(device_info_request)
response = self._msg_query(request, expected_response="device_info")
result = {
"name":
response.device_info.name,
"version":
response.device_info.version,
"initialized":
response.device_info.initialized,
"mnemonic_passphrase_enabled":
response.device_info.mnemonic_passphrase_enabled,
"monotonic_increments_remaining":
response.device_info.monotonic_increments_remaining,
}
if self.version >= semver.VersionInfo(9, 6, 0):
result["securechip_model"] = response.device_info.securechip_model
return result
def generate():
for folder in glob.glob("./test/*/www"):
# Create new repo
root = folder
repo = Repo.init(root)
# Generate some random commit
for i in range(0x10):
filename = "index-{:02d}.php".format(i)
filepath = os.path.join(root, filename)
with open(filepath, 'w') as f:
f.write(random_string())
repo.index.add([filename])
repo.index.commit("create {}".format(filename))
version = semver.VersionInfo(0, 0, 1)
# Create branches
branch_names = ["dev", "master", "main", "hotfix",
"release", "fix", "wip", "issue", "daily"]
for branch_name in branch_names:
try:
branch = repo.create_head(branch_name)
# Create tags
if random.choice([True, False]):
version = random.choice(
[version.bump_patch, version.bump_minor, version.bump_major])()
repo.create_tag(
version,
ref=branch,
message="v{}".format(version)
)
except:
pass
# Generate some random files
unstashed = []
for i in range(0x10):
filename = "{}.php".format(random_string())
filepath = os.path.join(root, filename)
with open(filepath, 'w') as f:
f.write(random_string())
if random.choice([True, False]):
repo.index.add([filename])
else:
unstashed.append(filename)
# Create stashes
git = repo.git
git.stash()
# Second stash
for i in unstashed:
repo.index.add([i])
# Create stashes
git = repo.git
git.stash()
def parse_version(version_str):
intel_version_style = re.search(r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$',
version_str)
major_minor_style = re.search(r'^(\d+)\.(\d+)$', version_str)
if intel_version_style:
major = intel_version_style.group(1)
minor = intel_version_style.group(2)
patchlevel = intel_version_style.group(3)
prerelease = intel_version_style.group(4)
ret = semver.VersionInfo(major, minor, patchlevel, prerelease)
elif major_minor_style:
major = major_minor_style.group(1)
minor = major_minor_style.group(2)
ret = semver.VersionInfo(major, minor, 0)
else:
ret = semver.VersionInfo.parse(version_str)
return ret
def _eth_coin(self, chain_id: int) -> "eth.ETHCoin.V":
"""Returns the deprecated `coin` enum value for a given chain_id. Only ETH, Ropsten and Rinkeby are converted, as these were the only supported networks up to v9.10.0. With v9.10.0, the chain ID is passed directly, and the `coin` field is ignored."""
if self.version < semver.VersionInfo(9, 10, 0):
return {
1: eth.ETHCoin.ETH,
3: eth.ETHCoin.RopstenETH,
4: eth.ETHCoin.RinkebyETH,
}[chain_id]
return eth.ETHCoin.ETH
def __init__(self,
device_info,
show_pairing_callback,
attestation_check_callback=None):
self.debug = False
serial_number = device_info["serial_number"]
self.version = parse_device_version(serial_number)
if self.version is None:
raise ValueError(f"Could not parse version from {serial_number}")
self.device = hid.device()
self.device.open_path(device_info["path"])
if self.version > semver.VersionInfo(1, 0, 0):
if attestation_check_callback is not None:
# Perform attestation
attestation_check_callback(self._perform_attestation())
# Invoke unlock workflow on the device.
# In version <=1.0.0, the device did this automatically.
self._query(OP_UNLOCK)
if self._query(OP_I_CAN_HAS_HANDSHAEK) != RESPONSE_SUCCESS:
raise Exception("Couldn't kick off handshake")
# init noise channel
noise = NoiseConnection.from_name(b"Noise_XX_25519_ChaChaPoly_SHA256")
noise.set_as_initiator()
dummy_private_key = os.urandom(32)
noise.set_keypair_from_private_bytes(Keypair.STATIC, dummy_private_key)
noise.set_prologue(b"Noise_XX_25519_ChaChaPoly_SHA256")
noise.start_handshake()
noise.read_message(self._query(noise.write_message()))
assert not noise.handshake_finished
send_msg = noise.write_message()
assert noise.handshake_finished
pairing_code = base64.b32encode(
noise.get_handshake_hash()).decode("ascii")
show_pairing_callback("{} {}\n{} {}".format(pairing_code[:5],
pairing_code[5:10],
pairing_code[10:15],
pairing_code[15:20]))
response = self._query(send_msg)
# Can be set to False if the remote static pubkey was previously confirmed.
pairing_verification_required_by_host = True
pairing_verification_required_by_device = response == b"\x01"
if pairing_verification_required_by_host or pairing_verification_required_by_device:
pairing_response = self._query(OP_I_CAN_HAS_PAIRIN_VERIFICASHUN)
if pairing_response == RESPONSE_SUCCESS:
pass
elif pairing_response == RESPONSE_FAILURE:
raise Exception("pairing rejected by the user")
else:
raise Exception("unexpected response")
self.noise = noise
def __set_version(self, prefix, version):
self.version_name = prefix
self.version_parsed = version
import semver
self.semver = str(semver.VersionInfo(*version))
stringified = self.semver
if prefix == BaseParser.SPEC_VERSION_2_PREFIX:
stringified = '%d.%d' % (version[0], version[1])
self.version = '%s %s' % (self.version_name, stringified)
def _encrypted_query(self, msg: bytes) -> bytes:
"""
Sends msg bytes and reads response bytes over an encrypted channel.
"""
encrypted_msg = self.noise.encrypt(msg)
if self.version >= semver.VersionInfo(4, 0, 0):
encrypted_msg = OP_NOISE_MSG + encrypted_msg
result = self.noise.decrypt(self._query(encrypted_msg))
assert isinstance(result, bytes)
return result
def extract_version(self, type, device_info, min_supported, max_supported,
level):
minor = level - min_supported
current = semver.VersionInfo(1, minor, 0)
# TODO: uncomment when compatiblity is broken
# min_compatibility = config['min_semver'] # e.g. "1.0.0"
# if semver.VersionInfo.parse(min_compatibility) > current:
# # TODO: coverage
# raise OutdatedBoxVersion(device_info)
# TODO: for now, BleBox assumes backward-compatibility
max_minor = max_supported - min_supported
latest_version = semver.VersionInfo(1, max_minor, 0)
# TODO: uncomment when backward compatiblity is broken
# if current > latest_version:
# raise UnsupportedAppVersion(device_info)
outdated = current < latest_version
return (current, outdated)
def get_version(db):
cursor = db.cursor()
cursor.execute('SHOW SERVER_VERSION;')
raw_version = cursor.fetchone()[0]
try:
version_parts = raw_version.split(' ')[0].split('.')
version = [int(part) for part in version_parts]
while len(version) < 3:
version.append(0)
return semver.VersionInfo(*version)
except Exception:
# Postgres might be in development, with format \d+[beta|rc]\d+
match = re.match(r'(\d+)([a-zA-Z]+)(\d+)', raw_version)
if match:
version = list(match.groups())
# We found a valid development version
if len(version) == 3:
# Replace development tag with a negative number to properly compare versions
version[1] = -1
version = [int(part) for part in version]
return semver.VersionInfo(*version)
raise Exception("Cannot determine which version is {}".format(raw_version))
def eth_sign_msg(self,
msg: bytes,
keypath: Sequence[int],
chain_id: int = 1) -> bytes:
"""
Signs message, the msg will be prefixed with "\x19Ethereum message\n" + len(msg) in the
hardware. 27 is added to the recID to denote an uncompressed pubkey.
"""
def format_as_uncompressed(sig: bytes) -> bytes:
# 27 is the magic constant to add to the recoverable ID to denote an uncompressed
# pubkey.
modified_signature = list(sig)
modified_signature[64] += 27
return bytes(modified_signature)
request = eth.ETHRequest()
# pylint: disable=no-member
request.sign_msg.CopyFrom(
eth.ETHSignMessageRequest(coin=self._eth_coin(chain_id),
chain_id=chain_id,
keypath=keypath,
msg=msg))
supports_antiklepto = self.version >= semver.VersionInfo(9, 5, 0)
if supports_antiklepto:
host_nonce = os.urandom(32)
request.sign_msg.host_nonce_commitment.commitment = antiklepto_host_commit(
host_nonce)
signer_commitment = self._eth_msg_query(
request, expected_response="antiklepto_signer_commitment"
).antiklepto_signer_commitment.commitment
request = eth.ETHRequest()
request.antiklepto_signature.CopyFrom(
antiklepto.AntiKleptoSignatureRequest(host_nonce=host_nonce))
signature = self._eth_msg_query(
request, expected_response="sign").sign.signature
antiklepto_verify(host_nonce, signer_commitment, signature[:64])
if self.debug:
print("Antiklepto nonce verification PASSED")
return format_as_uncompressed(signature)
signature = self._eth_msg_query(
request, expected_response="sign").sign.signature
return format_as_uncompressed(signature)
def eth_sign(self,
transaction: bytes,
keypath: Sequence[int],
chain_id: int = 1) -> bytes:
"""
transaction should be given as a full rlp encoded eth transaction.
"""
nonce, gas_price, gas_limit, recipient, value, data, _, _, _ = rlp.decode(
transaction)
request = eth.ETHRequest()
# pylint: disable=no-member
request.sign.CopyFrom(
eth.ETHSignRequest(
coin=self._eth_coin(chain_id),
chain_id=chain_id,
keypath=keypath,
nonce=nonce,
gas_price=gas_price,
gas_limit=gas_limit,
recipient=recipient,
value=value,
data=data,
))
supports_antiklepto = self.version >= semver.VersionInfo(9, 5, 0)
if supports_antiklepto:
host_nonce = os.urandom(32)
request.sign.host_nonce_commitment.commitment = antiklepto_host_commit(
host_nonce)
signer_commitment = self._eth_msg_query(
request, expected_response="antiklepto_signer_commitment"
).antiklepto_signer_commitment.commitment
request = eth.ETHRequest()
request.antiklepto_signature.CopyFrom(
antiklepto.AntiKleptoSignatureRequest(host_nonce=host_nonce))
signature = self._eth_msg_query(
request, expected_response="sign").sign.signature
antiklepto_verify(host_nonce, signer_commitment, signature[:64])
if self.debug:
print("Antiklepto nonce verification PASSED")
return signature
return self._eth_msg_query(request,
expected_response="sign").sign.signature
def calculate_version(base_major=1,
base_minor=0,
base_revision=0,
base_pre="alpha"):
"""Calculates a semver based on commit history and special flags in commit messages"""
major = base_major
minor = base_minor
patch = base_revision
pre = base_pre
commits = get_versionable_commits(REPO)
# Figure out what the current 'status' (prerelease) is
status_sets = [c for c in commits if is_status_set_command(c)]
if status_sets:
most_recent_message = status_sets[0].message.strip()
if most_recent_message.startswith("+setstatus "):
matcher = re.compile("\+setstatus (.+)")
pre = matcher.match(most_recent_message).group(1)
if most_recent_message == "+clearstatus":
pre = None
# If there are any +major in commit messages, increment the counter
major_incs = [c for c in commits if is_major_inc(c)]
if major_incs:
major += len(major_incs)
minor = 0
patch = 0
# If there are any +minor in commit messages, increment the counter
# We only care about commits after the last major increment
commits = list(itertools.takewhile(lambda c: not is_major_inc(c), commits))
minor_incs = [c for c in commits if is_minor_inc(c)]
if minor_incs:
minor += len(minor_incs)
patch = 0
# Now increment patch number for every commit since the last patch
commits = list(itertools.takewhile(lambda c: not is_minor_inc(c), commits))
commits = list(without_empty(commits))
patch = len(commits)
return "v" + str(semver.VersionInfo(major, minor, patch, pre))
def remove_old(
bucket: s3.Bucket,
prefix: pathlib.Path,
keep: int,
channel: str | None = None,
) -> None:
print("remove_old", bucket, prefix, keep, channel)
index: dict[str, dict[str, list[str]]] = {}
prefix_str = str(prefix) + "/"
for obj in bucket.objects.filter(Prefix=prefix_str):
if is_metadata_object(obj.key):
continue
metadata = get_metadata(bucket, obj.key)
if metadata["channel"] != channel:
continue
key = metadata["name"]
verslot = metadata.get("version_slot", "")
catver = (metadata["version_details"].get("metadata",
{}).get("catalog_version"))
if catver:
key = f"{key}-{catver}"
else:
key = f"{key}-{verslot}"
ver_details = metadata["version_details"]
version = semver.VersionInfo(
ver_details["major"],
ver_details["minor"],
ver_details["patch"] or 0,
".".join(f"{p['phase']}.{p['number']}"
for p in ver_details["prerelease"]),
)
index.setdefault(key, {}).setdefault(version, []).append(obj.key)
import pprint
pprint.pprint(index)
for _, versions in index.items():
sorted_versions = sorted(versions, reverse=True)
for ver in sorted_versions[keep:]:
for obj_key in versions[ver]:
print("Deleting outdated", obj_key)
bucket.objects.filter(Prefix=obj_key).delete()
def get_semver_from_source(data):
"""Given a dictionary of all version data available, determine the current version"""
# get the not-none values from data
known = {
key: data.get(alias)
for key, alias in config._forward_aliases.items()
if data.get(alias) is not None
}
_LOG.debug("valid, mapped keys: %r", known)
# prefer the non-strict field, if available, because it retains more information
potentials = [
known.get(Constants.VERSION_FIELD, None),
known.get(Constants.VERSION_STRICT_FIELD, None),
]
# build from components, if they're defined
from_components = {k: known.get(k) for k in SemVerSigFig if k in known}
try:
potentials.append(str(semver.VersionInfo(**from_components)))
except TypeError:
# we didn't have enough components
pass
versions = [
potential for potential in potentials if from_text_or_none(potential)
]
release_versions = {
semver.finalize_version(version)
for version in versions
}
if len(release_versions) > 1:
raise ValueError(
"conflicting versions within project: %s\nkeys were: %r" %
(release_versions, known))
if not versions:
_LOG.debug("key pairs found: \n%r", known)
raise ValueError("could not find existing semver")
result = None
if versions:
result = versions[0]
_LOG.info("latest version found in source: %r", result)
return result
def __init__(self, mlcube: t.Union[DictConfig, t.Dict], task: str) -> None:
super().__init__(mlcube, task)
try:
# Check version and log a warning message if fakeroot is used with singularity version < 3.5
version: semver.VersionInfo = get_singularity_version_info()
logger.info("SingularityRun singularity version = %s",
str(version))
if version < semver.VersionInfo(
major=3, minor=5) and "--fakeroot" in (
self.mlcube.runner.build_args or ""):
logger.warning(
"SingularityRun singularity version < 3.5, and it probably does not support --fakeroot "
"parameter that is present in MLCube configuration.")
except:
logger.warning(
"SingularityRun can't get singularity version (do you have singularity installed?)."
)
def _check_doxygen():
try:
with subprocess.Popen([DOXYGEN_PATH, "--version"],
stdout=subprocess.PIPE) as doxygen_exec:
version = doxygen_exec.stdout.read().decode("utf-8").strip()
try:
version = version.split()[0]
version = semver.VersionInfo.parse(version)
if version >= semver.VersionInfo(major=1, minor=8, patch=18):
return True
else:
return False
except:
return False
except FileNotFoundError as e:
log.warning(f"doxygen not found at '{DOXYGEN_PATH}' : {e}")
return False
def _get_rebuild_bundle_version(cls, version):
"""
Get a bundle version for the Freshmaker rebuild of the bundle image.
Examples:
1.2.3 => 1.2.3+0.$timestamp.p (no build ID and not a rebuild)
1.2.3+48273 => 1.2.3+48273.0.$timestamp.p (build ID and not a rebuild)
1.2.3+48273.0.1616457250.p => 1.2.3+48273.0.$timestamp.p (build ID and a rebuild)
:param str version: the version of the bundle image being rebuilt
:return: a tuple of the bundle version of the Freshmaker rebuild of the bundle image and
the suffix that was added by Freshmaker
:rtype: tuple(str, str)
"""
parsed_version = semver.VersionInfo.parse(version)
# Strip off the microseconds of the timestamp
timestamp = int(datetime.utcnow().timestamp())
new_fm_suffix = f'0.{timestamp}.p'
if parsed_version.build:
# Check if the bundle was a Freshmaker rebuild. Include .patched
# for backwards compatibility with the old suffix.
fm_suffix_search = re.search(
r'(?P<fm_suffix>0\.\d+\.(?:p|patched))$', parsed_version.build)
if fm_suffix_search:
fm_suffix = fm_suffix_search.groupdict()['fm_suffix']
# Get the build without the Freshmaker suffix. This may include a build ID
# from the original build before Freshmaker rebuilt it or be empty.
build_wo_fm_suffix = parsed_version.build[:-len(fm_suffix)]
new_build = f"{build_wo_fm_suffix}{new_fm_suffix}"
else:
# This was not previously rebuilt by Freshmaker so just append the suffix
# to the existing build ID with '.' separating it.
new_build = f"{parsed_version.build}.{new_fm_suffix}"
else:
# If there is existing build ID, then make the Freshmaker suffix the build ID
new_build = new_fm_suffix
# Don't use the replace method in order to support semver 2.8.1
new_version_dict = parsed_version._asdict()
new_version_dict["build"] = new_build
new_version = str(semver.VersionInfo(**new_version_dict))
return new_version, new_fm_suffix
