def install_setup(self, force=False):
"""
Install the template setup.py
"""
already_present = os.path.isfile(join(self.location, 'setup.py'))
if already_present and not force:
log.warning('setup.py already present, it will not be replaced')
log.warning('Run spvm install to force setup.py replacement')
return
if already_present:
log.fine('Creating setup.py.backup')
ioutils.copy(join(self.location, 'setup.py'), join(self.location, 'setup.py.backup'))
log.success('Copying seyup.py from template')
ioutils.copy(join(os.path.dirname(__file__), 'res', 'setup.py'), join(self.location, 'setup.py'))
def __call__(self, **kwargs):
"""
Run the pipeline
"""
splogger.fine(f'Starting pipeline {self.name}')
warnings.filterwarnings("ignore")
c = kfp.Client("https://kubflow.dvic.devinci.fr/pipeline")
self.res = c.create_run_from_pipeline_func(
self.func if self.func != None else self._generic_pipeline(),
kwargs,
self.run_name,
self.name,
namespace=self.namepsace)
splogger.success(f'Pipeline started', strong=True)
splogger.success(
f'Pipeline URL: https://kubflow.dvic.devinci.fr/_/pipeline/#/experiments/details/{self.res.run_id}'
)
return self
def _release_git(self, credentials = None):
if os.path.isfile('.git-credentials'):
os.remove('.git-credentials')
log.success('Removed dangling credential file')
# Commit version
commit_message = self.meta['project_vcs']['release']['commit_template'].replace('%s', self.meta['project_vcs']['version']).replace('"', '\\"').strip()
log.debug('Commit message: ' + commit_message)
ioutils.call_git('add .')
key = self.meta['project_vcs']['release']['git_signing_key']
if key != '':
log.success(Fore.GREEN + config.PADLOCK + 'Commit will be signed with ' + key)
ioutils.call_commit(commit_message, key=key)
# Tag version
tag = self.meta['project_vcs']['release']['tag_template'].replace('%s', self.meta['project_vcs']['version'])
ioutils.call_git('tag ' + ('' if key == '' else '-u ' + key + ' ') + '-m ' + tag + ' ' + tag)
log.success('Tagged: ' + tag)
try:
# Login
if credentials != None:
log.fine('Setting git credentials to temporary file')
u = urllib.parse.urlparse(self.meta['project_vcs']['code_repository'])
with open('.git-credentials', 'w+') as fh:
fh.write(u.scheme+'://'+credentials['login']+':'+credentials['password']+'@'+u.hostname+'\n')
ioutils.call_git(['config', 'credential.helper', 'store --file .git-credentials', '--replace-all'])
log.success('Credentials are set')
# Push
repo = self.meta['project_vcs']['code_repository']
log.success('Pushing to ' + repo)
ioutils.call_git('push ' + repo + ' --signed=if-asked')
log.success('Pushing tags')
ioutils.call_git('push ' + repo + ' --tags --signed=if-asked')
finally:
if credentials != None:
os.remove('.git-credentials')
log.success('Removed temporary credential file')
def up_version(self, kind): # FIXME other to 0
"""
Increase the version in the project meta base on the 'kind' instruction:
kind can be a str or a number
if a number, it is treated like an index for the version increment
if a string, it can be major, minor or patch
"""
log.fine('Increasing version (' + str(kind) + ')')
v = self.meta['project_vcs']['version']
v_ = [int(i) for i in v.split('.')]
while len(v_) < 3:
v_.insert(0, '0')
log.debug("Current version comphrension: " + str(v_))
if kind.isdigit():
kind = int(kind)
if kind < 0 or kind >= len(v_):
log.error('Unrecognized version changer: ' + str(kind))
v_[int(kind)] += 1
else:
kind = kind.lower()
if kind == 'patch':
index = len(v_) - 1
elif kind == 'major':
index = 0
elif kind == 'minor':
index = len(v_) - 2
elif kind == 'pass':
log.success('Version not changed')
return
else:
log.error('Unrecognized version changer: ' + str(kind))
exit(1)
v_[index] += 1
v_ = [v_[i] if i <= index else 0 for i in range(len(v_))]
self.meta['project_vcs']['version'] = '.'.join([str(i) for i in v_])
self.save_project_info()
log.success(v + ' -> ' + self.get_version())
def release(self, kind='pass'):
"""
Starts a release pipeline
"""
if self.get_project_status() != config.STATUS_PROJECT_INITIALIZED:
log.error('The project is not initialized')
log.error('Run spvm init first')
exit(1)
pipeline = []
log.fine('Calculating release pipeline')
pipeline.append(self.clear_build)
if config.config['update']:
pipeline.append(self.update_dependencies)
if config.config['repair']:
pipeline.append(self.repair)
pipeline.append(self.check_project)
if config.config['test']:
pipeline.append(self.run_test)
pipeline.append(self.up_version)
pipeline.append(self.populate_init)
pipeline.append(self.install_setup)
pipeline.append(self.publish)
NO = Fore.RED + 'NO' + Fore.RESET
MOCK = (
'' if not config.config['mock'] else ' ' +
Fore.LIGHTYELLOW_EX +
'(MOCK)' +
Fore.RESET)
YES = Fore.GREEN + 'YES' + Fore.RESET + MOCK
publish_context = self.detect_publish_context()
pipeline.append(Fore.CYAN + " - Git Publish:\t\t" + (YES if publish_context[0] else NO))
pipeline.append(Fore.CYAN + " - PyPi Publish:\t\t" + (YES if publish_context[1] else NO))
pipeline.append(Fore.CYAN + " - Docker Publish:\t" + (YES if publish_context[2] else NO))
log.success('Release pipeline is: ')
for f in pipeline:
if isinstance(f, str):
log.success(f)
continue
log.success(" -> " + f.__name__)
if not config.config['mock']:
log.warning(
Fore.YELLOW +
'The mock mode is not activated, this is for real !' +
Fore.RESET)
if config.config['ask']:
input('Press Enter to continue')
for f in pipeline:
if isinstance(f, str) or f.__name__ == 'wrapper':
continue
log.success('> ' + f.__name__)
if f.__name__ == 'up_version': # the only one to give parameters to
f(kind)
else:
f.__call__()
def check_packages(base_url='https://pypi.python.org/pypi/'):
log.fine('Checking packages in: ' + piptmp)
unchecked = 0
for f in os.listdir(piptmp):
try:
log.set_additional_info(f)
f_ = piptmp + os.sep + f
if not os.path.isfile(f_):
continue
splited = f.split('-')
log.debug('Checking ' + splited[0])
package_info = query_get(base_url + splited[0] + '/' +
splited[1] + '/json')
for f_info in package_info['releases'][splited[1]]:
if not os.path.isfile(
os.path.join(piptmp, f_info['filename'])):
continue
if md5(f_) != f_info['md5_digest']:
log.error('Hash do not match')
exit(1)
# log.success(Fore.GREEN+'Hash checked for '+f)
if not f_info['has_sig']:
log.debug(Fore.YELLOW + 'No signature provided for ' +
f_info['filename']) # FIXME throw?
unchecked += 1
continue
sig = query_get(f_info['url'] + '.asc', False)
log.debug('File: ' + f_info['filename'] +
' has signature:\n ' + sig.decode())
# Check
q = '' if log.get_verbose() else ' --quiet'
try:
call_gpg('--no-default-keyring --keyring tmp.gpg' + q +
' --auto-key-retrieve --verify - ' + f_,
inp=sig) # FIXME Only use known keys?
except CalledProcessError as er:
if er.returncode == 1:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Invalid signature for ' + f)
exit(1)
log.error('Could not check signature for ' + f + ' (' +
repr(er) + ')')
unchecked += 1
continue
log.success(Fore.GREEN + config.PADLOCK + ' File ' + f +
' is verified')
except KeyboardInterrupt:
exit(2)
except SystemExit as e:
raise e
except BaseException as be:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Failed to check ' + f + Fore.RESET)
log.error(repr(be))
log.warning(Fore.YELLOW + str(unchecked) +
' file(s) could not be verified')
def __enter__(self):
splogger.fine("Building pipeline")
DVICPipelineWrapper.current_pipeline = self
return self