def populate_init(self): """ Populate the <proj>/__init__.py with meta info """ log.debug('Populating the __init__') init_path = join(self.get_name().lower(), '__init__.py') with open(init_path, 'r') as fh: init_file = fh.read() def replace_or_create(key, value): nonlocal init_file pattern = "^" + key + " ?= ?.*" # key at the beginning of the line preg = re.compile(pattern, re.M) if len(preg.findall(init_file)) == 0: init_file += '\n' + key + ' = "' + value + '"' log.debug(key + ' not found in init') else: init_file = preg.sub(key + ' = "' + value + '"', init_file) log.debug('Found ' + key + ' and replaced with ' + value) replace_or_create('__name__', self.meta['project_info']['name']) replace_or_create('__version__', self.meta['project_vcs']['version']) replace_or_create('__author__', self.meta['project_authors'][0]['name']) replace_or_create('__url__', self.meta['project_info']['url']) replace_or_create('__email__', self.meta['project_authors'][0]['email']) os.remove(init_path) with open(init_path, 'w+') as fh: fh.write(init_file) log.success('Populated __init__.py')
def build(self): log.success('Building package in ./build') try: ioutils.call_python('', 'setup.py sdist -d build/dist bdist_wheel -d build/dist', stdout=subprocess.PIPE) except CalledProcessError as ex: log.error('Unable to build the package') log.error(repr(ex)) exit(1)
def ask_logins(): if os.path.isfile('.logins'): log.warning(Fore.YELLOW + 'The logins file already exist, overwrite it ?' + Fore.RESET) yes = input('Enter \'yes\' to overwrite: ') if yes != 'yes': return cr = {'creation_date': get_date(), 'data': {}} print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') print(Fore.YELLOW + '\tLogins configuration\n' + Fore.RESET) print( Fore.GREEN + 'You are going to be asked you credentials for this project\n' + 'They will be stored under .logins crypted with AES and .logins append to .gitignore' + Fore.RESET) yes = input('Do you wish to continue? (Enter "yes" to continue): ') if yes != 'yes': log.error('Cancelled') return print('') def ask_login(arr, component): print(Fore.LIGHTGREEN_EX + '\nNow configuring login for: ' + Fore.GREEN + component) print(Fore.RED + "Leave blank if Not Applicable" + Fore.RESET) login = input('Login: '******'': return password = getpass.getpass() arr[component] = {'login': login, 'password': password} # print(Fore.RED+'Git login: '******' Git login will no be asked, you are expected to use a credential helper for git ' # +'if you wish to automatically push'+Fore.RESET) # print(Fore.GREEN+'This setup can launch the git credential helper for you <3' # +'\n'+Fore.YELLOW+'WARNING: The credentials are stored in clear text'+Fore.RESET) # yes = input('Save git credentials? (Enter "yes" to continue): ') # if yes == 'yes': # call_git('config credential.helper store') # call_git('push') ask_login(cr['data'], 'git') ask_login(cr['data'], 'pypi') ask_login(cr['data'], 'docker') gpg = gnupg.GPG() gpg.encrypt(json.dumps(cr), (), symmetric=True, output='.logins') try: call_git('check-ignore .logins') except CalledProcessError: with open('.gitignore', 'a') as fh: fh.write('\.logins') log.success('Appened .logins to .gitignore')
def _pypi_upload(self, credentials = None): mock = config.config['mock'] # Upload if mock: rep = "https://test.pypi.org/legacy/" # FIXME put in config else: rep = self.meta['project_vcs']['pypi_repository'] log.success('Uploading to ' + rep) ioutils.call_twine('upload --repository-url ' + rep + ' ./build/dist/*' + ('' if credentials == None else ' -u '+credentials['login']+ ' -p '+credentials['password']))
def _release_pypi(self, sign=True, credentials = None): # 🔒 🔐 🔏 🔓 if self.meta['project_vcs']['pypi_repository'] == '': log.success('Nothing to push to pypi') return self.clear_build() self.build() if sign: self._sign_package() self._pypi_upload(credentials) self.clear_build()
def run_test(self): """ Run the tests with pytest """ try: ioutils.call_pytest(self.location) except CalledProcessError as ex: if ex.returncode == 5: log.warning('No tests were found') log.warning('This is not considered fatal but is VERY STRONGLY discouraged') log.warning('Resuming in 2 seconds') sleep(2) return log.error('Tests Failed') exit(1) log.success('Tests passed')
def install_setup(self, force=False): """ Install the template setup.py """ already_present = os.path.isfile(join(self.location, 'setup.py')) if already_present and not force: log.warning('setup.py already present, it will not be replaced') log.warning('Run spvm install to force setup.py replacement') return if already_present: log.fine('Creating setup.py.backup') ioutils.copy(join(self.location, 'setup.py'), join(self.location, 'setup.py.backup')) log.success('Copying seyup.py from template') ioutils.copy(join(os.path.dirname(__file__), 'res', 'setup.py'), join(self.location, 'setup.py'))
def __call__(self, **kwargs): """ Run the pipeline """ splogger.fine(f'Starting pipeline {self.name}') warnings.filterwarnings("ignore") c = kfp.Client("https://kubflow.dvic.devinci.fr/pipeline") self.res = c.create_run_from_pipeline_func( self.func if self.func != None else self._generic_pipeline(), kwargs, self.run_name, self.name, namespace=self.namepsace) splogger.success(f'Pipeline started', strong=True) splogger.success( f'Pipeline URL: https://kubflow.dvic.devinci.fr/_/pipeline/#/experiments/details/{self.res.run_id}' ) return self
def init(self): """ Create the spvm project folder and init a base project structure Guess project meta and asks for correction, write in metaFile """ os.makedirs(join(self.location, 'test'), exist_ok=True) os.makedirs(join(self.location, os.path.basename(self.location).lower()), exist_ok=True) self.meta = metautils.detect_project_meta(self.location) # self.print_project_meta() while True: metautils.prompt_project_info(self.location, self.meta) self.print_project_meta() if metautils.input_with_default(f'{Fore.CYAN}Is this correct? {Fore.RESET}[y/n]') in ('y', 'Y', '1', 'yes', 'Yes'): break self.save_project_info() log.success('Project initialized')
def up_version(self, kind): # FIXME other to 0 """ Increase the version in the project meta base on the 'kind' instruction: kind can be a str or a number if a number, it is treated like an index for the version increment if a string, it can be major, minor or patch """ log.fine('Increasing version (' + str(kind) + ')') v = self.meta['project_vcs']['version'] v_ = [int(i) for i in v.split('.')] while len(v_) < 3: v_.insert(0, '0') log.debug("Current version comphrension: " + str(v_)) if kind.isdigit(): kind = int(kind) if kind < 0 or kind >= len(v_): log.error('Unrecognized version changer: ' + str(kind)) v_[int(kind)] += 1 else: kind = kind.lower() if kind == 'patch': index = len(v_) - 1 elif kind == 'major': index = 0 elif kind == 'minor': index = len(v_) - 2 elif kind == 'pass': log.success('Version not changed') return else: log.error('Unrecognized version changer: ' + str(kind)) exit(1) v_[index] += 1 v_ = [v_[i] if i <= index else 0 for i in range(len(v_))] self.meta['project_vcs']['version'] = '.'.join([str(i) for i in v_]) self.save_project_info() log.success(v + ' -> ' + self.get_version())
def read_logins(): if os.path.isfile('.logins'): log.success(Fore.GREEN + config.PADLOCK + " Found crypted logins file" + Fore.RESET) gpg = gnupg.GPG() cr = None with open('.logins', 'r') as fh: cr = fh.read() crypt = None while True: passphrase = getpass.getpass('Passphrase for login file: ') crypt = gpg.decrypt(cr, passphrase=passphrase) if not crypt.ok: log.error( Fore.RED + config.PADLOCK + 'Could not unlock the logins file, is the passphrase correct?' + Fore.RESET) # raise ValueError('Could not uncrypt the login file') else: break cr = json.loads(crypt.data) log.success('Logins creation time: ' + cr['creation_date']) log.success(Fore.GREEN + config.OPEN_PADLOCK + ' Got logins for ' + ', '.join(cr['data']) + Fore.RESET) return NoFailReadOnlyDict(cr['data'], default=None) return None
def _show_docker_progress(obj): nonlocal status if 'errorDetail' in obj: log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET) raise docker.errors.DockerException(obj['errorDetail']['message']) if 'stream' in obj: for line in obj['stream'].split('\n'): if line == '': continue log.success(line.strip()) status.clear() return if 'status' in obj: if 'id' not in obj: log.success(obj['status']) return if len(status) == 0: print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET) s = obj['id'].strip() + ' ' + obj['status'] + '\t' if 'progress' in obj: s += obj['progress'] if obj['id'] not in status: status[obj['id']] = {'index': len(status) + 1, 'str': s} print(s) return status[obj['id']]['str'] = s print('\033[F' * (len(status) + 1)) for e in status: print('\033[K' + status[e]['str'])
def _sign_package(self): """ Add the signatures to the package before upload """ meta_key = self.meta['project_vcs']['release']['package_signing_key'] if meta_key == '': log.error(Fore.RED + config.OPEN_PADLOCK + ' No key provided for package signing' + Fore.RESET) return log.success('Signing the package with the key: ' + meta_key) try: for place in os.walk(join('.', 'build', 'dist')): for f in place[2]: self._sign_file(join(place[0], f), meta_key) except CalledProcessError as ex: log.error(Fore.RED + config.OPEN_PADLOCK + ' Could not sign the package' + Fore.RESET) log.error('The program will now stop, you can resume with: spvm publish pypi') log.error('When the issues are fixed') log.error(repr(ex)) exit(1) return log.success(Fore.GREEN + config.PADLOCK + ' Package Signed with key: ' + meta_key + Fore.RESET)
def _run(self, name): log.success('Running script: '+name) script = self.meta['scripts'][name] ioutils.call_with_stdout(['/bin/sh', '-c', script], stdout=None, stderr=None)
def _release_docker(self, credentials = None): log.success('Building Docker Image') client = docker.from_env() log.debug(json.dumps(client.version(), indent=4)) status = {} def _show_docker_progress(obj): nonlocal status if 'errorDetail' in obj: log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET) raise docker.errors.DockerException(obj['errorDetail']['message']) if 'stream' in obj: for line in obj['stream'].split('\n'): if line == '': continue log.success(line.strip()) status.clear() return if 'status' in obj: if 'id' not in obj: log.success(obj['status']) return if len(status) == 0: print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET) s = obj['id'].strip() + ' ' + obj['status'] + '\t' if 'progress' in obj: s += obj['progress'] if obj['id'] not in status: status[obj['id']] = {'index': len(status) + 1, 'str': s} print(s) return status[obj['id']]['str'] = s print('\033[F' * (len(status) + 1)) for e in status: print('\033[K' + status[e]['str']) # print('\n'*i, end = '') # FIXME choose dockerfile rep = self.meta['project_vcs']['docker_repository'] log.success('Image repo: ' + rep) if hasattr(client, 'api'): client = client.api g = client.build(tag=rep, path='.', dockerfile='Dockerfile') for line in g: _show_docker_progress(json.loads(line.decode())) if config.config['mock']: log.warning(Fore.YELLOW + 'Mock mode: not pushing' + Fore.RESET) return if credentials != None: try: client.login(credentials['login'], credentials['password']) except docker.errors.APIError as excep: log.error('Cannot login: '******'Logged in as '+credentials['login']) log.success('Pushing image') for line in client.push(rep, stream=True): _show_docker_progress(json.loads(line.decode()))
def clearup(): shutil.rmtree(piptmp, True) log.success('Cleaned temporary download directory')
def _release_git(self, credentials = None): if os.path.isfile('.git-credentials'): os.remove('.git-credentials') log.success('Removed dangling credential file') # Commit version commit_message = self.meta['project_vcs']['release']['commit_template'].replace('%s', self.meta['project_vcs']['version']).replace('"', '\\"').strip() log.debug('Commit message: ' + commit_message) ioutils.call_git('add .') key = self.meta['project_vcs']['release']['git_signing_key'] if key != '': log.success(Fore.GREEN + config.PADLOCK + 'Commit will be signed with ' + key) ioutils.call_commit(commit_message, key=key) # Tag version tag = self.meta['project_vcs']['release']['tag_template'].replace('%s', self.meta['project_vcs']['version']) ioutils.call_git('tag ' + ('' if key == '' else '-u ' + key + ' ') + '-m ' + tag + ' ' + tag) log.success('Tagged: ' + tag) try: # Login if credentials != None: log.fine('Setting git credentials to temporary file') u = urllib.parse.urlparse(self.meta['project_vcs']['code_repository']) with open('.git-credentials', 'w+') as fh: fh.write(u.scheme+'://'+credentials['login']+':'+credentials['password']+'@'+u.hostname+'\n') ioutils.call_git(['config', 'credential.helper', 'store --file .git-credentials', '--replace-all']) log.success('Credentials are set') # Push repo = self.meta['project_vcs']['code_repository'] log.success('Pushing to ' + repo) ioutils.call_git('push ' + repo + ' --signed=if-asked') log.success('Pushing tags') ioutils.call_git('push ' + repo + ' --tags --signed=if-asked') finally: if credentials != None: os.remove('.git-credentials') log.success('Removed temporary credential file')
def check_packages(base_url='https://pypi.python.org/pypi/'): log.fine('Checking packages in: ' + piptmp) unchecked = 0 for f in os.listdir(piptmp): try: log.set_additional_info(f) f_ = piptmp + os.sep + f if not os.path.isfile(f_): continue splited = f.split('-') log.debug('Checking ' + splited[0]) package_info = query_get(base_url + splited[0] + '/' + splited[1] + '/json') for f_info in package_info['releases'][splited[1]]: if not os.path.isfile( os.path.join(piptmp, f_info['filename'])): continue if md5(f_) != f_info['md5_digest']: log.error('Hash do not match') exit(1) # log.success(Fore.GREEN+'Hash checked for '+f) if not f_info['has_sig']: log.debug(Fore.YELLOW + 'No signature provided for ' + f_info['filename']) # FIXME throw? unchecked += 1 continue sig = query_get(f_info['url'] + '.asc', False) log.debug('File: ' + f_info['filename'] + ' has signature:\n ' + sig.decode()) # Check q = '' if log.get_verbose() else ' --quiet' try: call_gpg('--no-default-keyring --keyring tmp.gpg' + q + ' --auto-key-retrieve --verify - ' + f_, inp=sig) # FIXME Only use known keys? except CalledProcessError as er: if er.returncode == 1: log.error(Fore.RED + config.OPEN_PADLOCK + ' Invalid signature for ' + f) exit(1) log.error('Could not check signature for ' + f + ' (' + repr(er) + ')') unchecked += 1 continue log.success(Fore.GREEN + config.PADLOCK + ' File ' + f + ' is verified') except KeyboardInterrupt: exit(2) except SystemExit as e: raise e except BaseException as be: log.error(Fore.RED + config.OPEN_PADLOCK + ' Failed to check ' + f + Fore.RESET) log.error(repr(be)) log.warning(Fore.YELLOW + str(unchecked) + ' file(s) could not be verified')
def release(self, kind='pass'): """ Starts a release pipeline """ if self.get_project_status() != config.STATUS_PROJECT_INITIALIZED: log.error('The project is not initialized') log.error('Run spvm init first') exit(1) pipeline = [] log.fine('Calculating release pipeline') pipeline.append(self.clear_build) if config.config['update']: pipeline.append(self.update_dependencies) if config.config['repair']: pipeline.append(self.repair) pipeline.append(self.check_project) if config.config['test']: pipeline.append(self.run_test) pipeline.append(self.up_version) pipeline.append(self.populate_init) pipeline.append(self.install_setup) pipeline.append(self.publish) NO = Fore.RED + 'NO' + Fore.RESET MOCK = ( '' if not config.config['mock'] else ' ' + Fore.LIGHTYELLOW_EX + '(MOCK)' + Fore.RESET) YES = Fore.GREEN + 'YES' + Fore.RESET + MOCK publish_context = self.detect_publish_context() pipeline.append(Fore.CYAN + " - Git Publish:\t\t" + (YES if publish_context[0] else NO)) pipeline.append(Fore.CYAN + " - PyPi Publish:\t\t" + (YES if publish_context[1] else NO)) pipeline.append(Fore.CYAN + " - Docker Publish:\t" + (YES if publish_context[2] else NO)) log.success('Release pipeline is: ') for f in pipeline: if isinstance(f, str): log.success(f) continue log.success(" -> " + f.__name__) if not config.config['mock']: log.warning( Fore.YELLOW + 'The mock mode is not activated, this is for real !' + Fore.RESET) if config.config['ask']: input('Press Enter to continue') for f in pipeline: if isinstance(f, str) or f.__name__ == 'wrapper': continue log.success('> ' + f.__name__) if f.__name__ == 'up_version': # the only one to give parameters to f(kind) else: f.__call__()