def semver_compatible(compare_ver, min_version, max_version):
"""Method to check if all version numbers are semver compatible"""
return (
VersionInfo.isvalid(compare_ver)
and VersionInfo.isvalid(min_version)
and VersionInfo.isvalid(max_version)
)
def get_from_python(self, lookup: str, key=None):
"""Retrieves the version of a Python package by importing the package
and printing the ``__version__`` value.
Returns `None` if the package version cannot be printed.
"""
if key is None:
key = lookup
self.contdata[key] = self.ERROR
p = subprocess.run( # nosec B602
"python -c 'import {} as x; print(x.__version__);'".format(lookup),
shell=True,
capture_output=True,
check=False,
)
result = p.stdout.decode("utf-8")
if p.returncode != 0:
logger.info(
"Failed to get '%s' package version from python: %s",
lookup,
result,
)
return
for line in result.split("\n"):
if VersionInfo.isvalid(line):
result = line
break
self.contdata[key] = result.strip()
def isValidSemver(version):
"""Semantic version number - determines whether the version is qualified.
The format is MAJOR.Minor.PATCH, more with https://semver.org
"""
if version and isinstance(version, string_types):
return VersionInfo.isvalid(version)
return False
def update(ctx, all): # pragma: no cover
"""
Checks for out-of-date modules on the connected CIRCUITPYTHON device, and
prompts the user to confirm updating such modules.
"""
logger.info("Update")
# Grab out of date modules.
modules = [
m for m in find_modules(ctx.obj["DEVICE_PATH"], get_bundles_list())
if m.outofdate
]
if modules:
click.echo("Found {} module[s] needing update.".format(len(modules)))
if not all:
click.echo("Please indicate which modules you wish to update:\n")
for module in modules:
update_flag = all
if VERBOSE:
click.echo("Device version: {}, Bundle version: {}".format(
module.device_version, module.bundle_version))
if isinstance(
module.bundle_version,
str) and not VersionInfo.isvalid(module.bundle_version):
click.secho(
f"WARNING: Library {module.name} repo has incorrect __version__"
"\n\tmetadata. Circup will assume it needs updating."
"\n\tPlease file an issue in the library repo.",
fg="yellow",
)
if module.repo:
click.secho(f"\t{module.repo}", fg="yellow")
if not update_flag:
if module.mpy_mismatch:
click.secho(
f"WARNING: '{module.name}': mpy format doesn't match the"
" device's Circuitpython version. Updating is required.",
fg="yellow",
)
update_flag = click.confirm("Do you want to update?")
elif module.major_update:
update_flag = click.confirm((
"'{}' is a Major Version update and may contain breaking "
"changes. Do you want to update?".format(module.name)))
else:
update_flag = click.confirm("Update '{}'?".format(
module.name))
if update_flag:
# pylint: disable=broad-except
try:
module.update()
click.echo("Updated {}".format(module.name))
except Exception as ex:
logger.exception(ex)
click.echo(
"Something went wrong, {} (check the logs)".format(
str(ex)))
# pylint: enable=broad-except
return
click.echo("None of the modules found on the device need an update.")
def test_validate_semver(self):
# It follows strictly the 2.0.0 version of the SemVer scheme.
# For more information: https://semver.org/spec/v2.0.0.html
assert isinstance(__version__, str)
assert VersionInfo.isvalid(__version__), (
"Invalid semantic-version."
"For more information: https://semver.org/spec/v2.0.0.html")
assert __version__ == PYCLN_METADATA["version"]
class TestMetadata:
"""`__init__.py`/`pyproject.toml` test case."""
def test_pyproject_path(self):
assert PYPROJECT_PATH.is_file()
assert str(PYPROJECT_PATH).endswith("pyproject.toml")
def test_doc(self):
assert isinstance(__doc__, str)
assert len(__doc__) > 30, "Too short description!"
assert len(__doc__) <= 100, "Too long description!"
assert __doc__ == PYCLN_METADATA["description"]
def test_name(self):
assert isinstance(__name__, str)
assert __name__ == PYCLN_METADATA["name"] == "pycln"
def test_validate_semver(self):
# It follows strictly the 2.0.0 version of the SemVer scheme.
# For more information: https://semver.org/spec/v2.0.0.html
assert isinstance(__version__, str)
assert VersionInfo.isvalid(__version__), (
"Invalid semantic-version."
"For more information: https://semver.org/spec/v2.0.0.html")
assert __version__ == PYCLN_METADATA["version"]
@pytest.mark.skipif(
getenv("CI", "false") != "true"
or not VersionInfo.isvalid(__version__),
reason="Invalid semantic-version.",
)
def test_compare_semver(self):
# It follows strictly the 2.0.0 version of the SemVer scheme.
# For more information: https://semver.org/spec/v2.0.0.html
pycln_json = requests.get(PYCLN_PYPI_JSON_URL, timeout=5)
latest_version = pycln_json.json()["info"]["version"]
current_version = VersionInfo.parse(__version__)
assert current_version.compare(latest_version) > -1, (
f"Current version ({current_version}) can't be less than the "
f"latest released ({PYCLN_PYPI_URL}) version ({latest_version})!")
@pytest.mark.parametrize(
"value, expec_err, expec_exit_code",
[
pytest.param(True, Exit, 0, id="value=True"),
pytest.param(False, sysu.Pass, None, id="value=False"),
],
)
def test_version_callback(self, value, expec_err, expec_exit_code):
with sysu.std_redirect(sysu.STD.OUT) as stream:
with pytest.raises(expec_err):
version_callback(value)
raise sysu.Pass()
stdout = stream.getvalue()
if expec_err is Exit:
assert __version__ in stdout
else:
assert not stdout
def dockerhub_versions(url: str) -> ArchVersions:
dh_res = requests.get(url)
dh_result = dh_res.json()
out: ArchVersions = {arch: [] for arch in ARCHITECTURES}
for release in dh_result["results"]:
version_str = release["name"].split("-")[0]
if VersionInfo.isvalid(version_str):
for image in release["images"]:
arch = image["architecture"]
if arch in ARCHITECTURES:
out[arch].append(to_version(version_str))
logging.debug("Dockerhub versions=%s", out)
return out
def is_valid(string: str) -> bool:
"""Return whether a version is valid.
Parameters
----------
string : `str`
The version as a string.
Returns
-------
result : `bool`
Whether the version is valid.
"""
version = string[1:] if string.startswith("v") else string
return VersionInfo.isvalid(version)
def __init__(self, *, workspace_id: str, name: str, terraform_version: str,
auto_apply: bool, is_locked: bool, working_directory: str,
agent_pool_id: str, execution_mode: str,
speculative: bool): # pragma: no cover
self.workspace_id = workspace_id
self.name = name
self.terraform_version = terraform_version
self.parsed_terraform_version: Optional[VersionInfo] = None
if VersionInfo.isvalid(terraform_version):
self.parsed_terraform_version: VersionInfo = VersionInfo.parse(
terraform_version)
self.is_auto_updating: bool = self.terraform_version == LATEST_VERSION
self.auto_apply = auto_apply
self.is_locked = is_locked
self.working_directory = working_directory
self.agent_pool_id = agent_pool_id
self.execution_mode = execution_mode
self.speculative = speculative
def sanitize_version(_ctx: Optional[click.Context],
_param: Optional[Union[click.Option, click.Parameter]],
value: str
) -> str:
"""Sanitize a version number by stripping git tag ref and leading "v".
To be used as the callback of a click option or parameter.
Args:
ctx: Click context object.
param: The click option or parameter the callback is being used with.
value: Value passed to the option or parameter from the CLI.
Returns:
str: The SemVer version number.
"""
version = value.replace('refs/tags/', '') # strip git ref
if version.startswith('v'): # strip leading "v"
version = version[1:]
if VersionInfo.isvalid(version): # valid SemVer
return version
raise ValueError(f'version of "{version}" does not follow SemVer')
def test_should_versioninfo_isvalid():
assert VersionInfo.isvalid("1.0.0") is True
assert VersionInfo.isvalid("foo") is False
def main(ids,arguments,nointeraction=False):
# ============== Get Data ===============================================
if not os.path.isfile('.'+os.sep+'sailboat.toml'):
print('Please create a config file with `sailboat wizard` first.')
sys.exit(0)
try:
data = toml.loads(open('.'+os.sep+'sailboat.toml').read())
except toml.decoder.TomlDecodeError as e:
print('Config error:\n\t'+str(e))
exit()
# ============== Get VersionInfo ===============================================
if 'latest_build' not in data:
data['latest_build'] = '0.0.1'
if len(ids) >= 2: #Something provided
if VersionInfo.isvalid(ids[1]):
version = ids[1]
elif ids[1].startswith('maj'):
version = str(VersionInfo.parse(data['latest_build']).bump_major())
elif ids[1].startswith('min'):
version = str(VersionInfo.parse(data['latest_build']).bump_minor())
elif ids[1].startswith('pat'):
version = str(VersionInfo.parse(data['latest_build']).bump_patch())
elif ids[1].startswith('pre') or ids[1].startswith('dev'):
version = str(VersionInfo.parse(data['latest_build']).bump_prerelease())
else:
print('Unknown version `{}`'.format(ids[1]))
sys.exit(0)
else:
try:
latestcommit = os.popen('git rev-parse --short HEAD').read().replace('\n','')
except KeyboardInterrupt:
latestcommit = "build"
if latestcommit in data['latest_build']:
version = str(VersionInfo.parse(data['latest_build']).bump_build())
else:
version = str(VersionInfo.parse(data['latest_build']).replace(build=latestcommit+".1"))
if compare(version,data['latest_build']) == -1 and not (ids[1].startswith('pre') or ids[1].startswith('dev')):
if input(f'\u001b[31mYou are building a version ({version}) that comes before the previously built version ({data["latest_build"]}). Do you wish to continue? [y/n] \u001b[0m')[0]=='n' or nointeraction:
print()
sys.exit(0)
print('\nPreparing to build version {}\n'.format(version))
# ============== Pre-build script ===============================================
if 'build_script' in data['build']:
try:
buildscript = __import__(data['build']['build_script'].replace('.py',''))
except BaseException as e:
print('Error with custom prebuild script:\n'+str(e))
pass
try:
newdata = buildscript.pre(version,data)
if isinstance(newdata,dict):
data = newdata
except BaseException as e:
print('Error with custom prebuild script:\n\t`'+str(e)+"`")
pass
# ============== Show what will happen ===============================================
print('This command will build the following:')
print('\t- Generate a correct directory structure, setup.py, .gitignore, etc...')
only=False
for x in arguments:
if '-only' in x[0]:
only=True
break
dopypi,dobrew,domac,dowin,doact,doset=False,False,False,False,False,False
if only:
if ('--pypi-only','') in arguments:
dopypi = True
elif ('--homebrew-only','') in arguments:
dobrew = True
elif ('--windows-only','') in arguments and sys.platform.startswith('win'):
dowin = True
elif ('--mac-only','') in arguments and sys.platform.startswith('darwin'):
domac = True
elif ('--actions-only','') in arguments:
doact = True
elif ('--setup-only','') in arguments:
doset = True
elif ('--unix-only','') in arguments:
donix = True
elif 'actions_only' in data['build'] and data['build']['actions_only'] and not ('CI' in os.environ and os.environ['CI']=="TRUE"):
dopypi = True
dobrew = True
dowin = False
domac = False
donix = False
doact = True
doset = True
else:
dopypi = True
dobrew = data['build']['homebrew']
dowin = data['build']['windows'] and sys.platform.startswith('win')
domac = data['build']['mac'] and sys.platform.startswith('darwin')
donix = data['build']['unix'] and sys.platform.startswith('l')
doact = data['build']['actions']
doset =True
if ('--no-installer','') in arguments:
doinstall=False
else:
doinstall=data['build']['installer']
if dopypi:
print('\t- A distributable Python module.')
if dobrew:
print('\t- A Homebrew formula.')
if dowin:
installer = " with a .msi installer." if doinstall else "."
print('\t- A Windows app'+installer)
if domac:
installer = " with a .dmg installer." if doinstall else "."
print('\t- A Mac app'+installer)
if donix:
print('\t- A Unix executable')
if doact:
print('\t- A GitHub Actions file for building Mac and Windows apps and publishing a Github release.')
if not nointeraction:
input('Press enter to continue.')
# ============== Create bin Script ===============================================
try:
os.mkdir('bin')
except:
pass
bins = []
for commandname in data['build']['commands'].keys():
if data['build']['commands'][commandname] == '':
open("bin"+os.sep+commandname,'w+').write(f"#!"+os.sep+"usr"+os.sep+"bin"+os.sep+f"env bash\npython3 -m {data['short_name']} $@")
bins.append('bin'+os.sep+commandname)
# ============== Get module names ===============================================
if 'no_import' not in data['resources']:
data['resources']['no_import'] = []
mods = []
for x in glob.glob(data['short_name']+os.sep+'*.py'):
f = open(x)
mods += re.findall('(?m)(?:from[ ]+(\S+)[ ]+)?import[ ]+\S+?:[ ]+as[ ]+\S+?[ ]*$',f.read())
f.close()
modules = []
for x in mods:
modules.append(x[1].split('.')[0])
for module in set(modules):
if module not in data['resources']['no_import'] and ( module!= data['short_name'] and module not in sys.builtin_module_names and module not in data['resources']['modules']):
print('Checking for {} on PyPi...'.format(module))
response = requests.get("https://pypi.python.org/pypi/{}/json".format(module))
if response.status_code == 200:
data['resources']['modules'].append(module)
else:
data['resources']['no_import'].append(module)
# ============== Generate setup.py ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating setup.py\u001b[0m')
if doset:
if 'custom_setup' in data:
cu=str(data['custom_setup'])
else:
cu=str({})
with open(prefix+os.sep+'setup.py.template') as datafile:
template = datafile.read()
entries = []
for commandname in data['build']['commands'].keys():
if data['build']['commands'][commandname]!="":
modname = ".".join(data['build']['commands'][commandname].split('.')[:-1])
funcname = data['build']['commands'][commandname].split('.')[-1]
entries.append(commandname+"="+data["short_name"]+"."+modname+":"+funcname)
try:
pyv = version.split('+')[0]
except:
pyv = version
setup = template.format(
**data,
**data['resources'],
cu=cu,
bins=bins,
version = pyv,
entry_points = entries
)
open('setup.py','w+').write(setup)
# ============== Generate directory structure ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating Directory Structure\u001b[0m')
if not os.path.isfile('.gitignore'):
open('.'+os.sep+'.gitignore','w+').write(open(prefix+os.sep+'gitignore.template').read().replace('/',os.sep))
source_dir = os.getcwd()
target_dir = data["short_name"]+os.sep
types = ('*.py',*data['resources']["data_files"])
file_names = []
for files in types:
file_names.extend(glob.glob(files))
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
try:
bs = data['build']['build_script']
except:
bs = "RANDOM-----edfskjsdhflkjdhflksdjhflkasjdhflkasjdhflkasjdhflkajsdhflkjadshf"
for file_name in file_names:
if file_name in ("setup.py","sailboat.toml",bs):
continue
shutil.move(os.path.join(source_dir, file_name), target_dir+os.sep+file_name)
for filename in glob.glob(target_dir+os.sep+'LICE*'):
shutil.copyfile(filename,'LICENSE')
if not os.path.isfile(target_dir+'__init__.py'):
open(target_dir+'__init__.py','w+').write('# This file must exist, empty or not')
if data['resources']['file']!="" and not os.path.isfile(data['short_name']+os.sep+'__main__.py'):
try:
os.rename(data['short_name']+os.sep+data['resources']['file'],data['short_name']+os.sep+'__main__.py')
open(data['short_name']+os.sep+data['resources']['file'],'w+').write('# Please edit __main__.py for the main code. Thanks!\n(you can delete this file.)')
except FileNotFoundError:
pass
# ============== Generate pypi files ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating PyPi files...\u001b[0m')
if dopypi:
try:
shutil.rmtree('dist')
except:
pass
os.system('python3 .'+os.sep+'setup.py bdist_wheel sdist -d dist'+os.sep+'pypi')
try:
shutil.rmtree('build')
except:
pass
for x in glob.glob('dist'+os.sep+'*.whl'):
os.rename(x,x.replace('dist'+os.sep,'dist'+os.sep+'pypi'+os.sep))
for x in glob.glob('*.egg-info'):
shutil.rmtree(x)
# ============== Generate homebrew file ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating Homebrew file...\u001b[0m')
if dobrew:
retmp = ' resource "{name}" do\n url "{url}"\n sha256 "{sha256}"\n end\n'
resources = ''
for modulename in data['resources']['modules']:
req = requests.get('https://pypi.org/pypi/{}/json'.format(modulename)).json()
versionPy = req['info']['version']
url = req['releases'][versionPy][0]['url']
sha256 = req['releases'][versionPy][0]['digests']['sha256']
if not (url.endswith('.tar.gz') or url.endswith('.zip')):
try:
url = req['releases'][versionPy][1]['url']
sha256 = req['releases'][versionPy][1]['digests']['sha256']
except:
continue
resources+=retmp.format(name=modulename,url=url,sha256=sha256)
os.makedirs('dist'+os.sep+'homebrew')
f = open('dist'+os.sep+'homebrew'+os.sep+'{name}.rb'.format(name=data['name']),'w+')
f.write(open(prefix+os.sep+'brew.rb').read().format(
**data,
resources2 = resources,
version = version
))
f.close()
# ============== Generate w/Pyinstaller ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating Pyinstaller files...\u001b[0m')
# domac = True
if dowin or domac or donix:
try:
import PyInstaller.__main__
except:
print('Installing PyInstaller...')
os.system('pip3 install pyinstaller')
import PyInstaller.__main__
mods = []
for x in data['resources']['modules']:
mods.append('--hidden-import')
mods.append(x)
datafiles = []
for x in data['resources']['data_files']:
for g in glob.glob(data['short_name']+os.sep+x):
datafiles.append('--add-data')
datafiles.append(g+os.pathsep+g.replace(data['short_name']+os.sep,''))
typ = '--nowindowed' if data['build']['type']=='1' else '--noconsole'
ico = ['--icon',data['resources']['icon']] if 'icon' in data['resources'] else []
options = [
data['short_name']+os.sep+'__main__.py',
'--onefile',
'--name',
data['name'] if ('-n','')not in arguments else 'app',
'--distpath',
'.'+os.sep+'dist'+os.sep+'pyinstaller',
*mods,
*datafiles,
typ,
*ico,
'--osx-bundle-identifier',
data['build']['bundle_id']
]
print(options)
PyInstaller.__main__.run(options)
try:
print('removing '+data['name']+".spec...")
os.remove(data['name']+".spec")
except:
print('removing app.spec...')
os.remove("app.spec")
# ============== Mac .app Bundle ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating Mac .app bundle...\u001b[0m')
if domac:
os.chdir('dist')
os.mkdir(data['name'])
os.chdir(data['name'])
os.mkdir('Contents')
os.chdir('Contents')
os.mkdir('MacOS')
os.mkdir('Resources')
infoPlist = open('Info.plist','w+')
infoPlist.write(open(prefix+'/info.plist.xml').read().format(
**data,
**data['build'],
version = version
))
infoPlist.close()
shutil.copy('./../../pyinstaller/'+data['name'],'MacOS')
os.chdir('./../../..')
os.rename('./dist/'+data['name'],'./dist/'+data['name']+".app")
else:
print('not generating mac .app bundle because on {} not mac.'.format(sys.platform))
# ============== Generate Installer Package ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating Installer Package...\u001b[0m')
if sys.platform.startswith('win') and doinstall and False:#WINDOWS
os.system('pip install distro')
os.system('pip install git+https://github.com/x24git/wixpy')
d = open(prefix+os.sep+'wixpy.template.json').read().format(
**data,
version=version,
icns=data['resources']['icon'],
keywo=", ".join(data['keywords'])
)
open('wixpy.json','w+').write(d)
print(d)
print('STARTING WIXPY...')
os.system('wix.py wixpy.json')
elif sys.platform.startswith('darwin'):#MAC
try:
import dmgbuild
except:
print('Installing dmgbuild')
os.system('pip3 install dmgbuild')
import dmgbuild
del dmgbuild
if not os.path.isdir('build'):
os.mkdir('build')
open('build'+os.sep+'settings.py','w+').write(open(prefix+os.sep+'settings.py.template').read().format(
**data,
version=version,
icns=data['resources']['icon'],
keywo=", ".join(data['keywords'])
))
os.system(f'cat build/settings.py;dmgbuild -s .{os.sep}build{os.sep}settings.py "{data["name"]} Installer" ./{data["name"]}.dmg')
else:
print(f'Installer creation not yet supported for {sys.platform}!')
# ============== Generate Github Actions Workflow ===============================================
print('\n\n\u001b[4m\u001b[1;36mGenerating GitHub Actions File...\u001b[0m')
if doact:
try:
oldact = open('.github'+os.sep+'workflows'+os.sep+'sailboat.yml').read().split('\n')[0]
except:
oldact="\n"
try:
f = open('.github'+os.sep+'workflows'+os.sep+'sailboat.yml','w+')
except:
os.system('mkdir -p .github'+os.sep+'workflows'+os.sep)
f = open('.github'+os.sep+'workflows'+os.sep+'sailboat.yml','w+')
newdata = open(prefix+os.sep+'sailboat.yml.template').read().format(
**data,
mac=""if data['build']['mac']else"#",
windows=""if data['build']['windows']else"#",
win_ext=".exe"if data['build']['installer']else".exe",
mac_ext=".dmg"if not data['build']['unix']else"",
u=""if data['build']['unix']else"#"
).replace('\t',' ')
f.write(newdata)
f.close()
newact = open('.github'+os.sep+'workflows'+os.sep+'sailboat.yml').read().split('\n')[0]
data['build']['actions_built_latest'] = newact != oldact
print(oldact)
print(newact)
# ============== Post build ===============================================
try:
newdata = buildscript.post(version,data)
if isinstance(newdata,dict):
data = newdata
except:
pass
# ============== Save VersionInfo ===============================================
data['latest_build'] = version
open('sailboat.toml','w+').write(toml.dumps(data))
os.system('python .'+os.sep+'setup.py develop')
def run(self, **kwargs):
manager = enlighten.get_manager()
term = blessed.Terminal()
if len(self.options) > 0 and self.options[0] == 'help':
print(
"usage: sail build [version (or) increment] [plugins ...]\n\tThis command builds your project using the "
"sailboat.toml file.\n\tValid options for version:\n\t\t- Valid semver.org string: set that as "
"version\n\t\t- `major`: increments the major version by one.\n\t\t- `minor`: increments the minor "
"version by one.\n\t\t- `patch`: increments the patch version by one.\n\t\t- `pre`: increments the "
"prerelease version by one.\n\t\t- None: increments build version by one.")
return
# Get Version =====================================================================================
if 'latest_build' not in self.data:
self.data['latest_build'] = '0.0.1'
if len(self.options) >= 1: # Something provided
if VersionInfo.isvalid(self.options[0]):
version = self.options[0]
elif self.options[0].startswith('maj'):
version = str(VersionInfo.parse(self.data['latest_build']).bump_major())
elif self.options[0].startswith('min'):
version = str(VersionInfo.parse(self.data['latest_build']).bump_minor())
elif self.options[0].startswith('pat'):
version = str(VersionInfo.parse(self.data['latest_build']).bump_patch())
elif self.options[0].startswith('+') or self.options[0].startswith('build'):
version = str(VersionInfo.parse(self.data['latest_build']).bump_build())
else:
print('Unknown version `{}`'.format(self.options[0]))
return
if '.pre' in self.options[0] or '.dev' in self.options[0]:
version = str(VersionInfo.parse(self.data['latest_build']).bump_prerelease())
else:
try:
latestcommit = os.popen('git rev-parse --short HEAD').read().replace('\n', '')
except KeyboardInterrupt:
latestcommit = "build"
if latestcommit in self.data['latest_build']:
version = str(VersionInfo.parse(self.data['latest_build']).bump_build())
else:
version = str(VersionInfo.parse(self.data['latest_build']).replace(build=latestcommit + ".1"))
if compare(version, self.data['latest_build']) == -1 and not (
self.options[0].startswith('pre') or self.options[0].startswith('dev')):
if input(
term.red + f'You are building a version ({version}) that comes before the previously built version ({self.data["latest_build"]}). Do you wish to continue? [y/n]' + term.normal)[
0] == 'n' or ('-y' in self.options or '--no-interaction' in self.options):
print()
return
status_format = '{program}{fill}{current}{fill}{version}'
status_bar = manager.status_bar(status_format=status_format, color='white_on_blue', program=self.data['name'],
current='building directory structure', version=version)
print('\nPreparing to build version {}\n'.format(version))
self.data['latest_build'] = version
if len(self.options[1:]) == 0:
notdones = [*self.data['build'].keys()]
else:
notdones = self.options[1:]
progress_bar = manager.counter(total=len(notdones) + 3, desc='Build', unit='jobs', color="grey")
prebuild = progress_bar.add_subcounter('white')
postbuild = progress_bar.add_subcounter('darkgrey')
# =====================================================================================
if not os.path.isfile('.gitignore'):
open('.' + os.sep + '.gitignore', 'w+', encoding="utf8").write(
self.getResource('resources' + os.sep + 'gitignore.template').read().replace('/', os.sep))
source_dir = os.getcwd()
target_dir = self.data["short_name"] + os.sep
types = ('*.py', *self.data['resources']["data_files"])
file_names = []
for files in types:
file_names.extend(glob.glob(files))
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
for file_name in file_names:
if file_name in ("setup.py", "sailboat.toml", self.data['resources']['file']):
continue
shutil.move(os.path.join(source_dir, file_name), target_dir + os.sep + file_name)
for filename in glob.glob(target_dir + os.sep + 'LICE*'):
shutil.copyfile(filename, 'LICENSE')
if not os.path.isfile(target_dir + '__init__.py'):
open(target_dir + '__init__.py', 'w+', encoding="utf8").write('# This file must exist, empty or not')
if self.data['resources']['file'] != "" and not os.path.isfile(
self.data['short_name'] + os.sep + '__main__.py') and os.path.isfile(self.data['resources']['file']):
try:
os.rename(self.data['resources']['file'], self.data['short_name'] + os.sep + '__main__.py')
except:
pass
time.sleep(0.1);
status_bar.update(current="scanning imports")
prebuild.update()
# =====================================================================================
print('Scanning module imports...')
if 'no_import' not in self.data['resources']:
self.data['resources']['no_import'] = []
mods = []
for x in glob.glob(self.data['short_name'] + os.sep + '*.py'):
f = open(x, encoding="utf8")
b = f.read()
f.close()
mods += re.findall('^import[ ]+(.*)', b, re.M)
mods += re.findall('^from[ ]+(.*) import', b, re.M)
modules = []
for x in set(mods):
modules.append(x.split('.')[0])
for module in set(modules):
if module not in self.data['resources']['no_import'] and (
module != self.data['short_name'] and module not in sys.builtin_module_names and module not in
self.data['resources']['modules']):
print('Checking for {} on PyPi...'.format(module))
response = requests.get("https://pypi.python.org/pypi/{}/json".format(module))
if response.status_code == 200:
self.data['resources']['modules'].append(module)
else:
self.data['resources']['no_import'].append(module)
time.sleep(0.1);
status_bar.update(current="removing previous builds")
prebuild.update()
# =====================================================================================
try:
shutil.rmtree('dist')
except FileNotFoundError:
pass
dones = []
for build_plugin in progress_bar(notdones):
if build_plugin not in self.options and (
'_run' in self.data['build'][build_plugin] and not self.data['build'][build_plugin]['_run']):
continue
if build_plugin in dones:
continue
if build_plugin not in self.data['build'].keys():
continue
elif '_needs' in self.data['build'][build_plugin]:
if isinstance(self.data['build'][build_plugin]['_needs'], str):
self.data['build'][build_plugin]['_needs'] = [self.data['build'][build_plugin]['_needs']]
for x in self.data['build'][build_plugin]['_needs']:
if x not in dones:
notdones.append(build_plugin)
build_plugin = x
print(term.cyan + term.underline + build_plugin + term.normal + term.nounderline + "\n\n")
time.sleep(0.2);
status_bar.update(current=build_plugin)
try:
plugin_type, job = get_plugin(build_plugin, plugin_type="build")
job = job(
data=self.data,
options=[],
name=build_plugin,
prefix=self.prefix,
version=version
)
except PluginNotFound:
sys.exit(f'You seem to have added the {build_plugin} plugin, but it does not appear to be installed!')
try:
job.run()
except KeyboardInterrupt:
print('\n\nUser has aborted at step {}.\n\n'.format(build_plugin))
sys.exit(0)
except BaseException as error:
print('\n\nError at step {}:\n\n\t{}\n\n'.format(build_plugin, self.red(traceback.print_exc())))
sys.exit(1)
self.data[job._type][build_plugin] = job.data[job._type][build_plugin]
dones.append(build_plugin)
time.sleep(0.1);
status_bar.update(current="running develop")
print(self.section('Finishing up...'))
os.system('python3 setup.py develop')
postbuild.update()
print(self.section('Built files:'))
for x in glob.glob(f'.{os.sep}dist{os.sep}*{os.sep}*') +\
glob.glob(f'.{os.sep}dist{os.sep}*'):
print(x)
time.sleep(0.2);
status_bar.update(current='Finished Build!')
manager.stop()
def run(self, plugins, **kwargs):
if len(self.options) > 0 and self.options[0] == 'help':
print(
"usage: sail build [version (increment)]\n\tThis command builds your project using the sailboat.toml file.\n\tValid options for version:\n\t\t- Valid semver.org string: set that as version\n\t\t- `major`: increments the major version by one.\n\t\t- `minor`: increments the minor version by one.\n\t\t- `patch`: increments the patch version by one.\n\t\t- `pre`: increments the prerelease version by one.\n\t\t- None: increments build version by one."
)
return
#=====================================================================================
if 'latest_build' not in self.data:
self.data['latest_build'] = '0.0.1'
if len(self.options) >= 1: #Something provided
if VersionInfo.isvalid(self.options[0]):
version = self.options[0]
elif self.options[0].startswith('maj'):
version = str(
VersionInfo.parse(self.data['latest_build']).bump_major())
elif self.options[0].startswith('min'):
version = str(
VersionInfo.parse(self.data['latest_build']).bump_minor())
elif self.options[0].startswith('pat'):
version = str(
VersionInfo.parse(self.data['latest_build']).bump_patch())
elif self.options[0].startswith(
'pre') or self.options[0].startswith('dev'):
version = str(
VersionInfo.parse(
self.data['latest_build']).bump_prerelease())
elif self.options[0].startswith('+') or self.options[0].startswith(
'build'):
version = str(
VersionInfo.parse(self.data['latest_build']).bump_build())
else:
print('Unknown version `{}`'.format(self.options[0]))
return
else:
try:
latestcommit = os.popen(
'git rev-parse --short HEAD').read().replace('\n', '')
except KeyboardInterrupt:
latestcommit = "build"
if latestcommit in self.data['latest_build']:
version = str(
VersionInfo.parse(self.data['latest_build']).bump_build())
else:
version = str(
VersionInfo.parse(self.data['latest_build']).replace(
build=latestcommit + ".1"))
if compare(version, self.data['latest_build']) == -1 and not (
self.options[0].startswith('pre')
or self.options[0].startswith('dev')):
if input(
f'\u001b[31mYou are building a version ({version}) that comes before the previously built version ({self.data["latest_build"]}). Do you wish to continue? [y/n] \u001b[0m'
)[0] == 'n' or ('-y' in self.options
or '--no-interaction' in self.options):
print()
return
print('\nPreparing to build version {}\n'.format(version))
self.data['latest_build'] = version
#=====================================================================================
print(
'\n\n\u001b[4m\u001b[1;36mGenerating Directory Structure\u001b[0m')
print(
'This step will generate a correct directory structure for your project.'
)
if not os.path.isfile('.gitignore'):
open('.' + os.sep + '.gitignore', 'w+').write(
self.getResource('resources' + os.sep +
'gitignore.template').read().replace(
'/', os.sep))
source_dir = os.getcwd()
target_dir = self.data["short_name"] + os.sep
types = ('*.py', *self.data['resources']["data_files"])
file_names = []
for files in types:
file_names.extend(glob.glob(files))
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
for file_name in file_names:
if file_name in ("setup.py", "sailboat.toml",
self.data['resources']['file']):
continue
shutil.move(os.path.join(source_dir, file_name),
target_dir + os.sep + file_name)
for filename in glob.glob(target_dir + os.sep + 'LICE*'):
shutil.copyfile(filename, 'LICENSE')
if not os.path.isfile(target_dir + '__init__.py'):
open(target_dir + '__init__.py',
'w+').write('# This file must exist, empty or not')
if self.data['resources']['file'] != "" and not os.path.isfile(
self.data['short_name'] + os.sep +
'__main__.py') and os.path.isfile(
self.data['resources']['file']):
try:
os.rename(self.data['resources']['file'],
self.data['short_name'] + os.sep + '__main__.py')
except:
pass
#=====================================================================================
print('\n\n\u001b[4m\u001b[1;36mFetching Modules:\u001b[0m')
print('Scanning module imports...')
if 'no_import' not in self.data['resources']:
self.data['resources']['no_import'] = []
mods = []
for x in glob.glob(self.data['short_name'] + os.sep + '*.py'):
f = open(x)
b = f.read()
f.close()
mods += re.findall('^import[ ]+(.*)', b, re.M)
mods += re.findall('^from[ ]+(.*) import', b, re.M)
modules = []
for x in set(mods):
modules.append(x.split('.')[0])
for module in set(modules):
if module not in self.data['resources']['no_import'] and (
module != self.data['short_name']
and module not in sys.builtin_module_names
and module not in self.data['resources']['modules']):
print('Checking for {} on PyPi...'.format(module))
response = requests.get(
"https://pypi.python.org/pypi/{}/json".format(module))
if response.status_code == 200:
self.data['resources']['modules'].append(module)
else:
self.data['resources']['no_import'].append(module)
#=====================================================================================
try:
shutil.rmtree('dist')
except FileNotFoundError:
pass
dones = []
if len(self.options[1:]) == 0:
notdones = [*self.data['build'].keys()]
else:
notdones = self.options[1:]
for build_plugin in notdones:
if build_plugin not in self.options and (
'_run' in self.data['build'][build_plugin]
and not self.data['build'][build_plugin]['_run']):
continue
if build_plugin in dones:
continue
if build_plugin not in self.data['build'].keys():
continue
elif '_needs' in self.data['build'][build_plugin]:
if isinstance(self.data['build'][build_plugin]['_needs'], str):
self.data['build'][build_plugin]['_needs'] = [
self.data['build'][build_plugin]['_needs']
]
for x in self.data['build'][build_plugin]['_needs']:
if x not in dones:
notdones.append(build_plugin)
build_plugin = x
print('\n\u001b[4m\u001b[1;36m{}:\u001b[0m'.format(
build_plugin.title()))
try:
dist = plugins['build'][build_plugin]['dist']
job = pkg_resources.load_entry_point(dist, 'sailboat_plugins',
build_plugin)
job = job(data=self.data,
options=[],
name=build_plugin,
prefix=self.prefix,
version=version)
except KeyError as plugin:
sys.exit(
f'You seem to have added the {plugin} plugin, but it does not appear to be installed!'
)
try:
job.run()
except KeyboardInterrupt:
print('\n\nUser has aborted at step {}.\n\n'.format(
build_plugin))
return
except BaseException as error:
print('\n\nError at step {}:\n\n\t{}\n\n'.format(
build_plugin, self.red(error)))
return
self.data[job._type][build_plugin] = job.data[
job._type][build_plugin]
dones.append(build_plugin)
print(self.section('Finishing up...'))
os.system('python3 setup.py develop')
print(self.section('Built files:'))
for x in glob.glob(f'.{os.sep}dist{os.sep}*{os.sep}*') + glob.glob(
f'.{os.sep}dist{os.sep}*'):
if os.path.isfile(x):
print(x)
async def run(self):
logger = self.runtime.logger
# Load group config and releases.yml
logger.info("Loading build data...")
group_config = await util.load_group_config(self.group, self.assembly, env=self._doozer_env_vars)
releases_config = await util.load_releases_config(self._doozer_working_dir / "ocp-build-data")
if releases_config.get("releases", {}).get(self.assembly) is None:
raise ValueError(f"To promote this release, assembly {self.assembly} must be explictly defined in releases.yml.")
permits = util.get_assembly_promotion_permits(releases_config, self.assembly)
# Get release name
assembly_type = util.get_assembly_type(releases_config, self.assembly)
release_name = util.get_release_name(assembly_type, self.group, self.assembly, self.release_offset)
# Ensure release name is valid
if not VersionInfo.isvalid(release_name):
raise ValueError(f"Release name `{release_name}` is not a valid semver.")
logger.info("Release name: %s", release_name)
self._slack_client.bind_channel(release_name)
slack_response = await self._slack_client.say(f"Promoting release `{release_name}` @release-artists")
slack_thread = slack_response["message"]["ts"]
justifications = []
try:
# Get arches
arches = self.arches or group_config.get("arches", [])
arches = list(set(map(brew_arch_for_go_arch, arches)))
if not arches:
raise ValueError("No arches specified.")
# Get previous list
upgrades_str: Optional[str] = group_config.get("upgrades")
if upgrades_str is None and assembly_type != assembly.AssemblyTypes.CUSTOM:
raise ValueError(f"Group config for assembly {self.assembly} is missing the required `upgrades` field. If no upgrade edges are expected, please explicitly set the `upgrades` field to empty string.")
previous_list = list(map(lambda s: s.strip(), upgrades_str.split(","))) if upgrades_str else []
# Ensure all versions in previous list are valid semvers.
if any(map(lambda version: not VersionInfo.isvalid(version), previous_list)):
raise ValueError("Previous list (`upgrades` field in group config) has an invalid semver.")
# Check for blocker bugs
if self.skip_blocker_bug_check or assembly_type in [assembly.AssemblyTypes.CANDIDATE, assembly.AssemblyTypes.CUSTOM]:
logger.info("Blocker Bug check is skipped.")
else:
logger.info("Checking for blocker bugs...")
# TODO: Needs an option in releases.yml to skip this check
try:
await self.check_blocker_bugs()
except VerificationError as err:
logger.warn("Blocker bugs found for release: %s", err)
justification = self._reraise_if_not_permitted(err, "BLOCKER_BUGS", permits)
justifications.append(justification)
logger.info("No blocker bugs found.")
# If there are CVEs, convert RHBAs to RHSAs and attach CVE flaw bugs
impetus_advisories = group_config.get("advisories", {})
futures = []
for impetus, advisory in impetus_advisories.items():
if not advisory:
continue
if advisory < 0: # placeholder advisory id is still in group config?
raise ValueError("Found invalid %s advisory %s", impetus, advisory)
logger.info("Attaching CVE flaws for %s advisory %s...", impetus, advisory)
futures.append(self.attach_cve_flaws(advisory))
try:
await asyncio.gather(*futures)
except ChildProcessError as err:
logger.warn("Error attaching CVE flaw bugs: %s", err)
justification = self._reraise_if_not_permitted(err, "CVE_FLAWS", permits)
justifications.append(justification)
# Attempt to move all advisories to QE
futures = []
for impetus, advisory in impetus_advisories.items():
if not advisory:
continue
logger.info("Moving advisory %s to QE...", advisory)
futures.append(self.change_advisory_state(advisory, "QE"))
try:
await asyncio.gather(*futures)
except ChildProcessError as err:
logger.warn("Error moving advisory %s to QE: %s", advisory, err)
# Ensure the image advisory is in QE (or later) state.
image_advisory = impetus_advisories.get("image", 0)
errata_url = ""
if assembly_type == assembly.AssemblyTypes.STANDARD:
if image_advisory <= 0:
err = VerificationError(f"No associated image advisory for {self.assembly} is defined.")
justification = self._reraise_if_not_permitted(err, "NO_ERRATA", permits)
justifications.append(justification)
else:
logger.info("Verifying associated image advisory %s...", image_advisory)
image_advisory_info = await self.get_advisory_info(image_advisory)
try:
self.verify_image_advisory(image_advisory_info)
live_id = self.get_live_id(image_advisory_info)
assert live_id
errata_url = f"https://access.redhat.com/errata/{live_id}" # don't quote
except VerificationError as err:
logger.warn("%s", err)
justification = self._reraise_if_not_permitted(err, "INVALID_ERRATA_STATUS", permits)
justifications.append(justification)
# Verify attached bugs
advisories = list(filter(lambda ad: ad > 0, impetus_advisories.values()))
if advisories:
if self.skip_attached_bug_check:
logger.info("Skip checking attached bugs.")
else:
logger.info("Verifying attached bugs...")
try:
await self.verify_attached_bugs(advisories)
except ChildProcessError as err:
logger.warn("Error verifying attached bugs: %s", err)
justification = self._reraise_if_not_permitted(err, "ATTACHED_BUGS", permits)
justifications.append(justification)
# Promote release images
futures = []
metadata = {}
description = group_config.get("description")
if description:
logger.warning("The following description message will be included in the metadata of release image: %s", description)
metadata["description"] = str(description)
if errata_url:
metadata["url"] = errata_url
reference_releases = util.get_assmebly_basis(releases_config, self.assembly).get("reference_releases", {})
tag_stable = assembly_type in [assembly.AssemblyTypes.STANDARD, assembly.AssemblyTypes.CANDIDATE]
release_infos = await self.promote_all_arches(release_name, arches, previous_list, metadata, reference_releases, tag_stable)
self._logger.info("All release images for %s have been promoted.", release_name)
# Wait for release controllers
pullspecs = list(map(lambda r: r["image"], release_infos))
if not tag_stable:
self._logger.warning("This release will not appear on release controllers. Pullspecs: %s", release_name, ", ".join(pullspecs))
await self._slack_client.say(f"Release {release_name} is ready. It will not appear on the release controllers. Please tell the user to manually pull the release images: {', '.join(pullspecs)}", slack_thread)
else: # Wait for release images to be accepted by the release controllers
self._logger.info("All release images for %s have been successfully promoted. Pullspecs: %s", release_name, ", ".join(pullspecs))
# check if release is already accepted (in case we timeout and run the job again)
accepted = []
for arch in arches:
go_arch_suffix = util.go_suffix_for_arch(arch)
release_stream = f"4-stable{go_arch_suffix}"
accepted.append(await self.is_accepted(release_name, arch, release_stream))
if not all(accepted):
self._logger.info("Determining upgrade tests...")
test_commands = self._get_upgrade_tests_commands(release_name, previous_list)
message = f"""A new release `{release_name}` is ready and needs some upgrade tests to be triggered.
Please open a chat with @cluster-bot and issue each of these lines individually:
{os.linesep.join(test_commands)}
"""
await self._slack_client.say(message, slack_thread)
self._logger.info("Waiting for release images for %s to be accepted by the release controller...", release_name)
futures = []
for arch in arches:
go_arch_suffix = util.go_suffix_for_arch(arch)
release_stream = f"4-stable{go_arch_suffix}"
futures.append(self.wait_for_stable(release_name, arch, release_stream))
try:
await asyncio.gather(*futures)
except RetryError as err:
message = f"Timeout waiting for release to be accepted by the release controllers: {err}"
self._logger.error(message)
self._logger.exception(err)
raise TimeoutError(message)
self._logger.info("All release images for %s have been accepted by the release controllers.", release_name)
message = f"Release `{release_name}` has been accepted by the release controllers."
await self._slack_client.say(message, slack_thread)
# Send image list
if not image_advisory:
self._logger.warning("No need to send an advisory image list because this release doesn't have an image advisory.")
elif assembly_type == assembly.AssemblyTypes.CANDIDATE:
self._logger.warning("No need to send an advisory image list for a candidate release.")
elif self.skip_image_list:
self._logger.warning("Skip sending advisory image list")
else:
self._logger.info("Gathering and sending advisory image list...")
mail_dir = self._working_dir / "email"
await self.send_image_list_email(release_name, image_advisory, mail_dir)
self._logger.info("Advisory image list sent.")
except Exception as err:
self._logger.exception(err)
error_message = f"Error promoting release {release_name}: {err}\n {traceback.format_exc()}"
message = f"Promoting release {release_name} failed with: {error_message}"
await self._slack_client.say(message, slack_thread)
raise
# Print release infos to console
data = {
"group": self.group,
"assembly": self.assembly,
"type": assembly_type.value,
"name": release_name,
"content": {},
"justifications": justifications,
}
if image_advisory > 0:
data["advisory"] = image_advisory
if errata_url:
data["live_url"] = errata_url
for arch, release_info in zip(arches, release_infos):
data["content"][arch] = {
"pullspec": release_info["image"],
"digest": release_info["digest"],
}
from_release = release_info.get("references", {}).get("metadata", {}).get("annotations", {}).get("release.openshift.io/from-release")
if from_release:
data["content"][arch]["from_release"] = from_release
rhcos = next((t for t in release_info.get("references", {}).get("spec", {}).get("tags", []) if t["name"] == "machine-os-content"), None)
if rhcos:
rhcos_version = rhcos["annotations"]["io.openshift.build.versions"].split("=")[1] # machine-os=48.84.202112162302-0 => 48.84.202112162302-0
data["content"][arch]["rhcos_version"] = rhcos_version
json.dump(data, sys.stdout)
