def populate_init(self):
""" Populate the <proj>/__init__.py with meta info """
log.debug('Populating the __init__')
init_path = join(self.get_name().lower(), '__init__.py')
with open(init_path, 'r') as fh:
init_file = fh.read()
def replace_or_create(key, value):
nonlocal init_file
pattern = "^" + key + " ?= ?.*" # key at the beginning of the line
preg = re.compile(pattern, re.M)
if len(preg.findall(init_file)) == 0:
init_file += '\n' + key + ' = "' + value + '"'
log.debug(key + ' not found in init')
else:
init_file = preg.sub(key + ' = "' + value + '"', init_file)
log.debug('Found ' + key + ' and replaced with ' + value)
replace_or_create('__name__', self.meta['project_info']['name'])
replace_or_create('__version__', self.meta['project_vcs']['version'])
replace_or_create('__author__', self.meta['project_authors'][0]['name'])
replace_or_create('__url__', self.meta['project_info']['url'])
replace_or_create('__email__', self.meta['project_authors'][0]['email'])
os.remove(init_path)
with open(init_path, 'w+') as fh:
fh.write(init_file)
log.success('Populated __init__.py')
def build(self):
log.success('Building package in ./build')
try:
ioutils.call_python('', 'setup.py sdist -d build/dist bdist_wheel -d build/dist', stdout=subprocess.PIPE)
except CalledProcessError as ex:
log.error('Unable to build the package')
log.error(repr(ex))
exit(1)
def ask_logins():
if os.path.isfile('.logins'):
log.warning(Fore.YELLOW +
'The logins file already exist, overwrite it ?' +
Fore.RESET)
yes = input('Enter \'yes\' to overwrite: ')
if yes != 'yes':
return
cr = {'creation_date': get_date(), 'data': {}}
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
print(Fore.YELLOW + '\tLogins configuration\n' + Fore.RESET)
print(
Fore.GREEN +
'You are going to be asked you credentials for this project\n' +
'They will be stored under .logins crypted with AES and .logins append to .gitignore'
+ Fore.RESET)
yes = input('Do you wish to continue? (Enter "yes" to continue): ')
if yes != 'yes':
log.error('Cancelled')
return
print('')
def ask_login(arr, component):
print(Fore.LIGHTGREEN_EX + '\nNow configuring login for: ' +
Fore.GREEN + component)
print(Fore.RED + "Leave blank if Not Applicable" + Fore.RESET)
login = input('Login: '******'':
return
password = getpass.getpass()
arr[component] = {'login': login, 'password': password}
# print(Fore.RED+'Git login: '******' Git login will no be asked, you are expected to use a credential helper for git '
# +'if you wish to automatically push'+Fore.RESET)
# print(Fore.GREEN+'This setup can launch the git credential helper for you <3'
# +'\n'+Fore.YELLOW+'WARNING: The credentials are stored in clear text'+Fore.RESET)
# yes = input('Save git credentials? (Enter "yes" to continue): ')
# if yes == 'yes':
# call_git('config credential.helper store')
# call_git('push')
ask_login(cr['data'], 'git')
ask_login(cr['data'], 'pypi')
ask_login(cr['data'], 'docker')
gpg = gnupg.GPG()
gpg.encrypt(json.dumps(cr), (), symmetric=True, output='.logins')
try:
call_git('check-ignore .logins')
except CalledProcessError:
with open('.gitignore', 'a') as fh:
fh.write('\.logins')
log.success('Appened .logins to .gitignore')
def _pypi_upload(self, credentials = None):
mock = config.config['mock']
# Upload
if mock:
rep = "https://test.pypi.org/legacy/" # FIXME put in config
else:
rep = self.meta['project_vcs']['pypi_repository']
log.success('Uploading to ' + rep)
ioutils.call_twine('upload --repository-url ' + rep + ' ./build/dist/*' + ('' if credentials == None else ' -u '+credentials['login']+ ' -p '+credentials['password']))
def _release_pypi(self, sign=True, credentials = None):
# 🔒 🔐 🔏 🔓
if self.meta['project_vcs']['pypi_repository'] == '':
log.success('Nothing to push to pypi')
return
self.clear_build()
self.build()
if sign:
self._sign_package()
self._pypi_upload(credentials)
self.clear_build()
def run_test(self):
"""
Run the tests with pytest
"""
try:
ioutils.call_pytest(self.location)
except CalledProcessError as ex:
if ex.returncode == 5:
log.warning('No tests were found')
log.warning('This is not considered fatal but is VERY STRONGLY discouraged')
log.warning('Resuming in 2 seconds')
sleep(2)
return
log.error('Tests Failed')
exit(1)
log.success('Tests passed')
def install_setup(self, force=False):
"""
Install the template setup.py
"""
already_present = os.path.isfile(join(self.location, 'setup.py'))
if already_present and not force:
log.warning('setup.py already present, it will not be replaced')
log.warning('Run spvm install to force setup.py replacement')
return
if already_present:
log.fine('Creating setup.py.backup')
ioutils.copy(join(self.location, 'setup.py'), join(self.location, 'setup.py.backup'))
log.success('Copying seyup.py from template')
ioutils.copy(join(os.path.dirname(__file__), 'res', 'setup.py'), join(self.location, 'setup.py'))
def __call__(self, **kwargs):
"""
Run the pipeline
"""
splogger.fine(f'Starting pipeline {self.name}')
warnings.filterwarnings("ignore")
c = kfp.Client("https://kubflow.dvic.devinci.fr/pipeline")
self.res = c.create_run_from_pipeline_func(
self.func if self.func != None else self._generic_pipeline(),
kwargs,
self.run_name,
self.name,
namespace=self.namepsace)
splogger.success(f'Pipeline started', strong=True)
splogger.success(
f'Pipeline URL: https://kubflow.dvic.devinci.fr/_/pipeline/#/experiments/details/{self.res.run_id}'
)
return self
def init(self):
"""
Create the spvm project folder and init a base project structure
Guess project meta and asks for correction, write in metaFile
"""
os.makedirs(join(self.location, 'test'), exist_ok=True)
os.makedirs(join(self.location, os.path.basename(self.location).lower()), exist_ok=True)
self.meta = metautils.detect_project_meta(self.location)
# self.print_project_meta()
while True:
metautils.prompt_project_info(self.location, self.meta)
self.print_project_meta()
if metautils.input_with_default(f'{Fore.CYAN}Is this correct? {Fore.RESET}[y/n]') in ('y', 'Y', '1', 'yes', 'Yes'):
break
self.save_project_info()
log.success('Project initialized')
def up_version(self, kind): # FIXME other to 0
"""
Increase the version in the project meta base on the 'kind' instruction:
kind can be a str or a number
if a number, it is treated like an index for the version increment
if a string, it can be major, minor or patch
"""
log.fine('Increasing version (' + str(kind) + ')')
v = self.meta['project_vcs']['version']
v_ = [int(i) for i in v.split('.')]
while len(v_) < 3:
v_.insert(0, '0')
log.debug("Current version comphrension: " + str(v_))
if kind.isdigit():
kind = int(kind)
if kind < 0 or kind >= len(v_):
log.error('Unrecognized version changer: ' + str(kind))
v_[int(kind)] += 1
else:
kind = kind.lower()
if kind == 'patch':
index = len(v_) - 1
elif kind == 'major':
index = 0
elif kind == 'minor':
index = len(v_) - 2
elif kind == 'pass':
log.success('Version not changed')
return
else:
log.error('Unrecognized version changer: ' + str(kind))
exit(1)
v_[index] += 1
v_ = [v_[i] if i <= index else 0 for i in range(len(v_))]
self.meta['project_vcs']['version'] = '.'.join([str(i) for i in v_])
self.save_project_info()
log.success(v + ' -> ' + self.get_version())
def read_logins():
if os.path.isfile('.logins'):
log.success(Fore.GREEN + config.PADLOCK +
" Found crypted logins file" + Fore.RESET)
gpg = gnupg.GPG()
cr = None
with open('.logins', 'r') as fh:
cr = fh.read()
crypt = None
while True:
passphrase = getpass.getpass('Passphrase for login file: ')
crypt = gpg.decrypt(cr, passphrase=passphrase)
if not crypt.ok:
log.error(
Fore.RED + config.PADLOCK +
'Could not unlock the logins file, is the passphrase correct?'
+ Fore.RESET)
# raise ValueError('Could not uncrypt the login file')
else:
break
cr = json.loads(crypt.data)
log.success('Logins creation time: ' + cr['creation_date'])
log.success(Fore.GREEN + config.OPEN_PADLOCK + ' Got logins for ' +
', '.join(cr['data']) + Fore.RESET)
return NoFailReadOnlyDict(cr['data'], default=None)
return None
def _show_docker_progress(obj):
nonlocal status
if 'errorDetail' in obj:
log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET)
raise docker.errors.DockerException(obj['errorDetail']['message'])
if 'stream' in obj:
for line in obj['stream'].split('\n'):
if line == '':
continue
log.success(line.strip())
status.clear()
return
if 'status' in obj:
if 'id' not in obj:
log.success(obj['status'])
return
if len(status) == 0:
print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET)
s = obj['id'].strip() + ' ' + obj['status'] + '\t'
if 'progress' in obj:
s += obj['progress']
if obj['id'] not in status:
status[obj['id']] = {'index': len(status) + 1, 'str': s}
print(s)
return
status[obj['id']]['str'] = s
print('\033[F' * (len(status) + 1))
for e in status:
print('\033[K' + status[e]['str'])
def _sign_package(self):
"""
Add the signatures to the package before upload
"""
meta_key = self.meta['project_vcs']['release']['package_signing_key']
if meta_key == '':
log.error(Fore.RED + config.OPEN_PADLOCK + ' No key provided for package signing' + Fore.RESET)
return
log.success('Signing the package with the key: ' + meta_key)
try:
for place in os.walk(join('.', 'build', 'dist')):
for f in place[2]:
self._sign_file(join(place[0], f), meta_key)
except CalledProcessError as ex:
log.error(Fore.RED + config.OPEN_PADLOCK + ' Could not sign the package' + Fore.RESET)
log.error('The program will now stop, you can resume with: spvm publish pypi')
log.error('When the issues are fixed')
log.error(repr(ex))
exit(1)
return
log.success(Fore.GREEN + config.PADLOCK + ' Package Signed with key: ' + meta_key + Fore.RESET)
def _run(self, name):
log.success('Running script: '+name)
script = self.meta['scripts'][name]
ioutils.call_with_stdout(['/bin/sh', '-c', script], stdout=None, stderr=None)
def _release_docker(self, credentials = None):
log.success('Building Docker Image')
client = docker.from_env()
log.debug(json.dumps(client.version(), indent=4))
status = {}
def _show_docker_progress(obj):
nonlocal status
if 'errorDetail' in obj:
log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET)
raise docker.errors.DockerException(obj['errorDetail']['message'])
if 'stream' in obj:
for line in obj['stream'].split('\n'):
if line == '':
continue
log.success(line.strip())
status.clear()
return
if 'status' in obj:
if 'id' not in obj:
log.success(obj['status'])
return
if len(status) == 0:
print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET)
s = obj['id'].strip() + ' ' + obj['status'] + '\t'
if 'progress' in obj:
s += obj['progress']
if obj['id'] not in status:
status[obj['id']] = {'index': len(status) + 1, 'str': s}
print(s)
return
status[obj['id']]['str'] = s
print('\033[F' * (len(status) + 1))
for e in status:
print('\033[K' + status[e]['str'])
# print('\n'*i, end = '')
# FIXME choose dockerfile
rep = self.meta['project_vcs']['docker_repository']
log.success('Image repo: ' + rep)
if hasattr(client, 'api'):
client = client.api
g = client.build(tag=rep, path='.', dockerfile='Dockerfile')
for line in g:
_show_docker_progress(json.loads(line.decode()))
if config.config['mock']:
log.warning(Fore.YELLOW + 'Mock mode: not pushing' + Fore.RESET)
return
if credentials != None:
try:
client.login(credentials['login'], credentials['password'])
except docker.errors.APIError as excep:
log.error('Cannot login: '******'Logged in as '+credentials['login'])
log.success('Pushing image')
for line in client.push(rep, stream=True):
_show_docker_progress(json.loads(line.decode()))
def clearup():
shutil.rmtree(piptmp, True)
log.success('Cleaned temporary download directory')
def _release_git(self, credentials = None):
if os.path.isfile('.git-credentials'):
os.remove('.git-credentials')
log.success('Removed dangling credential file')
# Commit version
commit_message = self.meta['project_vcs']['release']['commit_template'].replace('%s', self.meta['project_vcs']['version']).replace('"', '\\"').strip()
log.debug('Commit message: ' + commit_message)
ioutils.call_git('add .')
key = self.meta['project_vcs']['release']['git_signing_key']
if key != '':
log.success(Fore.GREEN + config.PADLOCK + 'Commit will be signed with ' + key)
ioutils.call_commit(commit_message, key=key)
# Tag version
tag = self.meta['project_vcs']['release']['tag_template'].replace('%s', self.meta['project_vcs']['version'])
ioutils.call_git('tag ' + ('' if key == '' else '-u ' + key + ' ') + '-m ' + tag + ' ' + tag)
log.success('Tagged: ' + tag)
try:
# Login
if credentials != None:
log.fine('Setting git credentials to temporary file')
u = urllib.parse.urlparse(self.meta['project_vcs']['code_repository'])
with open('.git-credentials', 'w+') as fh:
fh.write(u.scheme+'://'+credentials['login']+':'+credentials['password']+'@'+u.hostname+'\n')
ioutils.call_git(['config', 'credential.helper', 'store --file .git-credentials', '--replace-all'])
log.success('Credentials are set')
# Push
repo = self.meta['project_vcs']['code_repository']
log.success('Pushing to ' + repo)
ioutils.call_git('push ' + repo + ' --signed=if-asked')
log.success('Pushing tags')
ioutils.call_git('push ' + repo + ' --tags --signed=if-asked')
finally:
if credentials != None:
os.remove('.git-credentials')
log.success('Removed temporary credential file')
def check_packages(base_url='https://pypi.python.org/pypi/'):
log.fine('Checking packages in: ' + piptmp)
unchecked = 0
for f in os.listdir(piptmp):
try:
log.set_additional_info(f)
f_ = piptmp + os.sep + f
if not os.path.isfile(f_):
continue
splited = f.split('-')
log.debug('Checking ' + splited[0])
package_info = query_get(base_url + splited[0] + '/' +
splited[1] + '/json')
for f_info in package_info['releases'][splited[1]]:
if not os.path.isfile(
os.path.join(piptmp, f_info['filename'])):
continue
if md5(f_) != f_info['md5_digest']:
log.error('Hash do not match')
exit(1)
# log.success(Fore.GREEN+'Hash checked for '+f)
if not f_info['has_sig']:
log.debug(Fore.YELLOW + 'No signature provided for ' +
f_info['filename']) # FIXME throw?
unchecked += 1
continue
sig = query_get(f_info['url'] + '.asc', False)
log.debug('File: ' + f_info['filename'] +
' has signature:\n ' + sig.decode())
# Check
q = '' if log.get_verbose() else ' --quiet'
try:
call_gpg('--no-default-keyring --keyring tmp.gpg' + q +
' --auto-key-retrieve --verify - ' + f_,
inp=sig) # FIXME Only use known keys?
except CalledProcessError as er:
if er.returncode == 1:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Invalid signature for ' + f)
exit(1)
log.error('Could not check signature for ' + f + ' (' +
repr(er) + ')')
unchecked += 1
continue
log.success(Fore.GREEN + config.PADLOCK + ' File ' + f +
' is verified')
except KeyboardInterrupt:
exit(2)
except SystemExit as e:
raise e
except BaseException as be:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Failed to check ' + f + Fore.RESET)
log.error(repr(be))
log.warning(Fore.YELLOW + str(unchecked) +
' file(s) could not be verified')
def release(self, kind='pass'):
"""
Starts a release pipeline
"""
if self.get_project_status() != config.STATUS_PROJECT_INITIALIZED:
log.error('The project is not initialized')
log.error('Run spvm init first')
exit(1)
pipeline = []
log.fine('Calculating release pipeline')
pipeline.append(self.clear_build)
if config.config['update']:
pipeline.append(self.update_dependencies)
if config.config['repair']:
pipeline.append(self.repair)
pipeline.append(self.check_project)
if config.config['test']:
pipeline.append(self.run_test)
pipeline.append(self.up_version)
pipeline.append(self.populate_init)
pipeline.append(self.install_setup)
pipeline.append(self.publish)
NO = Fore.RED + 'NO' + Fore.RESET
MOCK = (
'' if not config.config['mock'] else ' ' +
Fore.LIGHTYELLOW_EX +
'(MOCK)' +
Fore.RESET)
YES = Fore.GREEN + 'YES' + Fore.RESET + MOCK
publish_context = self.detect_publish_context()
pipeline.append(Fore.CYAN + " - Git Publish:\t\t" + (YES if publish_context[0] else NO))
pipeline.append(Fore.CYAN + " - PyPi Publish:\t\t" + (YES if publish_context[1] else NO))
pipeline.append(Fore.CYAN + " - Docker Publish:\t" + (YES if publish_context[2] else NO))
log.success('Release pipeline is: ')
for f in pipeline:
if isinstance(f, str):
log.success(f)
continue
log.success(" -> " + f.__name__)
if not config.config['mock']:
log.warning(
Fore.YELLOW +
'The mock mode is not activated, this is for real !' +
Fore.RESET)
if config.config['ask']:
input('Press Enter to continue')
for f in pipeline:
if isinstance(f, str) or f.__name__ == 'wrapper':
continue
log.success('> ' + f.__name__)
if f.__name__ == 'up_version': # the only one to give parameters to
f(kind)
else:
f.__call__()