def prep_version_file_for_source_testing(build_type):
version_file = os.path.normpath('eng/versioning/version_' +
build_type.name + '.txt')
print('version_file=' + version_file)
file_changed = False
newlines = []
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
if hasattr(
module,
'current') and not module.current == module.dependency:
module.dependency = module.current
file_changed = True
newlines.append(module.string_for_version_file())
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
return file_changed
def increment_version_for_artifact(build_type, artifact_id):
if not build_type:
raise ValueError('build_type cannot be empty.')
if not artifact_id:
raise ValueError('artifact_id cannot be empty.')
version_file = os.path.normpath('eng/versioning/version_' +
build_type.name + '.txt')
print('version_file=' + version_file)
artifact_found = False
newlines = []
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
# Tick up the version here. If the version is already a pre-release
# version then just increment the revision. Otherwise increment the
# minor version, zero the patch and add "-beta.1" to the end
# https://github.com/Azure/azure-sdk/blob/master/docs/policies/releases.md#java
if module.artifact_id == artifact_id and hasattr(
module, 'current'):
artifact_found = True
vmatch = version_regex_named.match(module.current)
if (vmatch.group('prerelease') is not None):
prever = prerelease_regex_named.match(
vmatch.group('prerelease'))
rev = int(prever.group('revision'))
rev += 1
new_version = '{}.{}.{}-beta.{}'.format(
vmatch.group('major'), vmatch.group('minor'),
vmatch.group('patch'), str(rev))
else:
minor = int(vmatch.group('minor'))
minor += 1
new_version = '{}.{}.{}-beta.1'.format(
vmatch.group('major'), minor, 0)
print(
'artifact_id {}, previous version={}, new current version={}'
.format(artifact_id, module.current, new_version))
module.current = new_version
newlines.append(module.string_for_version_file())
if not artifact_found:
raise ValueError(
'artifact_id ({}) was not found in version file {}'.format(
artifact_id, version_file))
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
def increment_library_version(build_type, artifact_id, group_id):
version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
print('version_file=' + version_file)
library_to_update = group_id + ':' + artifact_id
artifact_found = False
newlines = []
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
# Tick up the version here. If the version is already a pre-release
# version then just increment the revision. Otherwise increment the
# minor version, zero the patch and add "-beta.1" to the end
# https://github.com/Azure/azure-sdk/blob/master/docs/policies/releases.md#java
if module.name == library_to_update and hasattr(module, 'current'):
artifact_found = True
vmatch = version_regex_named.match(module.current)
if (vmatch.group('prerelease') is not None):
prever = prerelease_regex_named.match(vmatch.group('prerelease'))
# This is the case where, somehow, the versioning verification has failed and
# the prerelease verification doesn't match "beta.X"
if prever is None:
raise ValueError('library_to_update ({}:{}) has an invalid prerelease version ({}) which should be of the format beta.X'.format(library_to_update, module.current, vmatch.group('prerelease')))
rev = int(prever.group('revision'))
rev += 1
new_version = '{}.{}.{}-beta.{}'.format(vmatch.group('major'), vmatch.group('minor'), vmatch.group('patch'), str(rev))
else:
minor = int(vmatch.group('minor'))
minor += 1
new_version = '{}.{}.{}-beta.1'.format(vmatch.group('major'), minor, 0)
# The dependency version only needs to be updated it if is different from the current version.
# This would be the case where a library hasn't been released yet and has been released (either GA or preview)
if (module.dependency != module.current):
print('library_to_update {}, previous dependency version={}, new dependency version={}'.format(library_to_update, module.dependency, module.current))
module.dependency = module.current
print('library_to_update {}, previous version={}, new current version={}'.format(library_to_update, module.current, new_version))
module.current = new_version
newlines.append(module.string_for_version_file())
if not artifact_found:
raise ValueError('library_to_update ({}) was not found in version file {}'.format(library_to_update, version_file))
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
def load_version_map_from_file(the_file, version_map):
with open(the_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
module = CodeModule(stripped_line)
# verify no duplicate entries
if (module.name in version_map):
raise ValueError(
'Version file: {0} contains a duplicate entry: {1}'.format(
the_file, module.name))
# verify that if the module is beta_ or unreleased_ that there's a matching non-beta_ or non-unreleased_ entry
if (module.name.startswith('beta_')
or module.name.startswith('unreleased_')):
tempName = module.name
if tempName.startswith('beta_'):
tempName = module.name[len('beta_'):]
else:
tempName = module.name[len('unreleased_'):]
# if there isn't a non beta or unreleased entry then raise an issue
if tempName not in version_map:
raise ValueError(
'Version file: {0} does not contain a non-beta or non-unreleased entry for beta_/unreleased_ library: {1}'
.format(the_file, module.name))
version_map[module.name] = module
def prep_version_file_for_source_testing(build_type):
version_file = os.path.normpath('eng/versioning/version_' +
build_type.name + '.txt')
print('version_file=' + version_file)
file_changed = False
# The version map is needed to get the 'current' version of any beta dependencies
# in order to update the beta_ version in the From Source runs
version_map = {}
newlines = []
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
if hasattr(
module,
'current') and not module.current == module.dependency:
module.dependency = module.current
file_changed = True
# In order to ensure that the From Source runs are effectively testing everything
# together using the latest source built libraries, ensure that the beta_ dependency's
# version is set
elif module.name.startswith('beta_'):
tempName = module.name[len('beta_'):]
if tempName in version_map:
# beta_ tags only have a dependency version, set that to
# the current version of the non-beta dependency
module.dependency = version_map[tempName].current
file_changed = True
else:
# if the beta_ dependency doesn't have a non-beta entry in the version file then this is an error
raise ValueError(
'prep_version_file_for_source_testing: beta library ({}) does not have a non-beta entry {} in version file {}'
.format(module.name, tempName, version_file))
version_map[module.name] = module
newlines.append(module.string_for_version_file())
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
return file_changed
def load_version_map_from_file(the_file, version_map):
with open(the_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
module = CodeModule(stripped_line)
version_map[module.name] = module
def verify_current_version_of_artifact(build_type, artifact_id, group_id):
version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
print('version_file=' + version_file)
library_to_update = group_id + ':' + artifact_id
artifact_found = False
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
else:
module = CodeModule(stripped_line)
# verify the current version of the artifact matches our version schema which is one
# of the following:
# <major>.<minor>.<patch/hotfix>
# <major>.<minor>.<patch/hotfix>-beta.<prerelease>
if module.name == library_to_update and hasattr(module, 'current'):
artifact_found = True
vmatch = version_regex_named.match(module.current)
temp_ver = '{}.{}.{}'.format(vmatch.group('major'), vmatch.group('minor'), vmatch.group('patch'))
# we should never have buildmetadata in our versioning scheme
if vmatch.group('buildmetadata') is not None:
raise ValueError('library ({}) version ({}) in version file ({}) is not a correct version to release. buildmetadata is set and should never be {}'.format(library_to_update, module.current, version_file, vmatch.group('buildmetadata')))
# reconstruct the version from the semver pieces and it should match exactly the current
# version in the module
# If there's a pre-release version it should be beta.X
if vmatch.group('prerelease') is not None:
prerel = vmatch.group('prerelease')
# this regex is looking for beta.X
if prerelease_regex_named.match(prerel) is None:
# if the build_type isn't data then error
if build_type.name.lower() != 'data':
raise ValueError('library ({}) version ({}) in version file ({}) is not a correct version to release. The accepted prerelease tag is (beta.X) and the current prerelease tag is ({})'.format(library_to_update, module.current, version_file, prerel))
else:
# verify that the prerelease tag is "beta" which is the only allowable thing for data track aside from beta.X
if prerelease_data_regex.match(prerel) is None:
raise ValueError('library ({}) version ({}) in version file ({}) is not a correct version to release. The accepted prerelease tags for data track are (beta) or (beta.X) and the current prerelease tag is ({})'.format(library_to_update, module.current, version_file, prerel))
# at this point the version is <major>.<minor>.<patch>-beta
temp_ver = '{}-{}'.format(temp_ver, str(prerel))
else:
prever = prerelease_regex_named.match(prerel)
rev = int(prever.group('revision'))
temp_ver = '{}-beta.{}'.format(temp_ver, str(rev))
# last but not least, for sanity verify that the version constructed from the
# semver pieces matches module's current version
if module.current != temp_ver:
raise ValueError('library ({}) version ({}) in version file ({}) does not match the version constructed from the semver pieces ({})'.format(library_to_update, module.current, version_file, temp_ver))
print('The version {} for {} looks good!'.format(module.current, module.name))
if not artifact_found:
raise ValueError('library ({}) was not found in version file {}'.format(library_to_update, version_file))
def set_dev_zero_version(build_type, build_qualifier):
version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
print('version_file=' + version_file)
# Assuming a build qualifier of the form: "dev.20200204.123"
# Converts "dev.20200204.123" -> "dev.20200204.0"
zero_qualifier = build_qualifier[:build_qualifier.rfind('.') + 1] + '0'
newlines = []
with open(version_file, encoding='utf8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
continue
module = CodeModule(stripped_line)
if module.name in items_we_should_not_update:
newlines.append(module.string_for_version_file())
continue
if module.name.startswith('unreleased_') or module.name.startswith('beta_'):
newlines.append(module.string_for_version_file())
continue
if hasattr(module, 'current'):
if 'dev' in module.current:
newlines.append(module.string_for_version_file())
continue
set_both = module.current == module.dependency
if '-' in module.current:
# if the module is 1.2.3-beta.x, strip off everything after the '-' and add the qualifier
module.current = module.current[:module.current.rfind('-') + 1] + zero_qualifier
else:
# if the module is a GA version 1.2.3, add '-' and the qualifier
module.current += '-' + zero_qualifier
# The resulting version must be a valid SemVer
match = version_regex_named.match(module.current)
if not match:
raise ValueError('{}\'s current version + build qualifier {} is not a valid semver version'.format(module.name, module.current + build_qualifier))
if set_both:
module.dependency = module.current
print(f'updating {module.name} to use dependency={module.dependency}, current={module.current}')
newlines.append(module.string_for_version_file())
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
def update_versions_file(update_type, build_type, build_qualifier,
artifact_id):
version_file = os.path.normpath('eng/versioning/version_' +
build_type.name + '.txt')
print('version_file=' + version_file)
newlines = []
with open(version_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
if not artifact_id or module.artifact_id == artifact_id:
if update_type == UpdateType.library or update_type == UpdateType.all:
if '-' in module.current:
module.current += "." + build_qualifier
else:
module.current += '-' + build_qualifier
match = version_regex.match(module.current)
if not match:
raise ValueError(
'{}\'s current version + build qualifier {} is not a valid semver version'
.format(module.name,
module.current + build_qualifier))
if update_type == UpdateType.external_dependency or update_type == UpdateType.all:
if '-' in module.dependency:
module.dependency += "." + build_qualifier
else:
module.dependency += '-' + build_qualifier
match = version_regex.match(module.dependency)
if not match:
raise ValueError(
'{}\'s dependency version + build qualifier {} is not a valid semver version'
.format(module.name,
module.dependency + build_qualifier))
newlines.append(module.string_for_version_file())
with open(version_file, 'w') as f:
for line in newlines:
f.write(line)
def load_version_map_from_file(the_file, version_map):
with open(the_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
module = CodeModule(stripped_line)
if (module.name in version_map):
raise ValueError('Version file: {0} contains a duplicate entry: {1}'.format(the_file, module.name))
version_map[module.name] = module
def load_version_overrides(the_file, version_map, overrides_name):
with open(the_file) as f:
data = json.load(f)
if overrides_name not in data:
raise ValueError('Version override name: {0} is not found in {1}'.format(overrides_name, the_file))
overrides = data[overrides_name]
for override in overrides:
if len(override) != 1:
raise ValueError('Expected exactly one module, but got: {0}'.format(override))
for module_name in override:
module_str = module_name + ";" + override[module_name]
module = CodeModule(module_str)
version_map[module.name] = module
break
def update_versions_file_for_nightly_devops(build_type, build_qualifier, artifact_id, group_id):
version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
print('version_file=' + version_file)
library_to_update = group_id + ':' + artifact_id
print('adding build_qualifier({}) to {}'.format(build_qualifier, library_to_update))
version_map = {}
newlines = []
artifact_found = False
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
# basically we don't want to change any of the parent versions here, only
# library versions
if module.name in items_we_should_not_update:
newlines.append(module.string_for_version_file())
continue
if library_to_update == module.name:
artifact_found = True
if hasattr(module, 'current'):
set_both = False
# In the case where the current and dependency are both equal then both
# need to be updated. In theory, this should only really happen when a
# new library has been added and has not yet been released but can also
# happen if a library has just been released and the devops build kicks
# of before the human submits the PR to increase the current version. Being
# that it's a devops build and both versions are being updated this should
# be OK.
if module.current == module.dependency:
set_both = True
if '-' in module.current:
# if the module is 1.2.3-beta.x, strip off everything after the '-' and add the qualifier
module.current = module.current[:module.current.rfind('-') + 1] + build_qualifier
else:
module.current += '-' + build_qualifier
match = version_regex_named.match(module.current)
if not match:
raise ValueError('{}\'s current version + build qualifier {} is not a valid semver version'.format(module.name, module.current + build_qualifier))
if set_both:
module.dependency = module.current
# If the library is not the update target and is unreleased, use
# a version range based on the build qualifier as the dependency
# version so that packages from this build that are released to
# the dev feed will depend on a version that should be found in
# the dev feed.
# This script runs once for each artifact so it makes no
# changes in the case where a dependency version has already
# been modified.
elif (module.name.startswith('unreleased_') or module.name.startswith('beta_')) and not module.dependency.startswith('['):
# Assuming a build qualifier of the form: "dev.20200204.1"
# Converts "dev.20200204.1" -> "dev.20200204."
unreleased_build_qualifier = build_qualifier[:build_qualifier.rfind('.') + 1]
if '-' in module.dependency:
# if the module is 1.2.3-beta.x, strip off everything after the '-' and add the qualifier
module.dependency = module.dependency[:module.dependency.rfind('-') + 1] + unreleased_build_qualifier
else:
module.dependency += '-' + unreleased_build_qualifier
print(f'updating unreleased/beta dependency {module.name} to use dependency version range: "{module.dependency}"')
version_map[module.name] = module
newlines.append(module.string_for_version_file())
if not artifact_found:
raise ValueError('library_to_update ({}) was not found in version file {}'.format(library_to_update, version_file))
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
def update_versions_file_for_nightly_devops(build_type, build_qualifier, artifact_id, group_id):
version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
print('version_file=' + version_file)
library_to_update = group_id + ':' + artifact_id
unreleased_library_to_update = "unreleased_" + library_to_update
print('adding build_qualifier({}) to {}'.format(build_qualifier, library_to_update))
version_map = {}
newlines = []
artifact_found = False
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
newlines.append(raw_line)
else:
module = CodeModule(stripped_line)
# basically we don't want to change any of the parent versions here, only
# library versions
if module.name in items_we_should_not_update:
newlines.append(module.string_for_version_file())
continue
if library_to_update == module.name or unreleased_library_to_update == module.name:
artifact_found = True
if hasattr(module, 'current'):
set_both = False
# In the case where the current and dependency are both equal then both
# need to be updated. In theory, this should only really happen when a
# new library has been added and has not yet been released but can also
# happen if a library has just been released and the devops build kicks
# of before the human submits the PR to increase the current version. Being
# that it's a devops build and both versions are being updated this should
# be OK.
if module.current == module.dependency:
set_both = True
if '-' in module.current:
module.current += "." + build_qualifier
else:
module.current += '-' + build_qualifier
match = version_regex_named.match(module.current)
if not match:
raise ValueError('{}\'s current version + build qualifier {} is not a valid semver version'.format(module.name, module.current + build_qualifier))
if set_both:
module.dependency = module.current
# we need to handle the unreleased dependency which should have the same version as the "current" dependency
# of the non-unreleased artifact
else:
if (module.name.startswith('unreleased_')):
# strip off the 'unreleased_' prepend to get the regular module entry
real_name = module.name.replace('unreleased_', '')
real_module = version_map[real_name]
module.dependency = real_module.current
else:
raise ValueError('{}\' does not have a current dependency and its groupId does not have the required unreleased_ prepend'.format(module.name))
version_map[module.name] = module
newlines.append(module.string_for_version_file())
if not artifact_found:
raise ValueError('library_to_update ({}) was not found in version file {}'.format(library_to_update, version_file))
with open(version_file, 'w', encoding='utf-8') as f:
for line in newlines:
f.write(line)
def verify_current_version_of_artifact(build_type, artifact_id):
if not build_type:
raise ValueError('build_type cannot be empty.')
if not artifact_id:
raise ValueError('artifact_id cannot be empty.')
version_file = os.path.normpath('eng/versioning/version_' +
build_type.name + '.txt')
print('version_file=' + version_file)
artifact_found = False
with open(version_file, encoding='utf-8') as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
else:
module = CodeModule(stripped_line)
# verify the current version of the artifact matches our version schema which is one
# of the following:
# <major>.<minor>.<patch/hotfix>
# <major>.<minor>.<patch/hotfix>-beta.<prerelease>
if module.artifact_id == artifact_id and hasattr(
module, 'current'):
artifact_found = True
vmatch = version_regex_named.match(module.current)
temp_ver = '{}.{}.{}'.format(vmatch.group('major'),
vmatch.group('minor'),
vmatch.group('patch'))
# we should never have buildmetadata in our versioning scheme
if vmatch.group('buildmetadata') is not None:
raise ValueError(
'artifact ({}) version ({}) in version file ({}) is not a correct version to release. buildmetadata is set and should never be {}'
.format(artifact_id, module.current, version_file,
vmatch.group('buildmetadata')))
# reconstruct the version from the semver pieces and it should match exactly the current
# version in the module
# If there's a pre-release version it should be beta.X
if vmatch.group('prerelease') is not None:
prerel = vmatch.group('prerelease')
if prerelease_regex_named.match(prerel) is None:
raise ValueError(
'artifact ({}) version ({}) in version file ({}) is not a correct version to release. The accepted prerelease tag is (beta.X) and the current prerelease tag is ({})'
.format(artifact_id, module.current,
version_file, prerel))
prever = prerelease_regex_named.match(prerel)
rev = int(prever.group('revision'))
temp_ver = '{}-beta.{}'.format(temp_ver, str(rev))
# last but not least, for sanity verify that the version constructed from the
# semver pieces matches module's current version
if module.current != temp_ver:
raise ValueError(
'artifact ({}) version ({}) in version file ({}) does not match the version constructed from the semver pieces ({})'
.format(artifact_id, module.current, version_file,
temp_ver))
print('The version {} for {} looks good!'.format(
module.current, module.name))
if not artifact_found:
raise ValueError(
'artifact_id ({}) was not found in version file {}'.format(
artifact_id, version_file))