def save_project_info(self): """ Write the current project info to the metaFile """ with open(self.projectMetaFile, 'w+') as fh: fh.write(json.dumps(self.meta, indent=4)) log.debug('Saved project meta')
def call_with_stdout(args, ignore_err=False, stdout=PIPE, inp=None, stderr=PIPE): with Popen(args.split(' ') if type(args) == str else args, stdout=stdout, stdin=PIPE if inp is not None else None, stderr=stderr) as proc: out, err = proc.communicate(input=inp) if proc.poll() != 0 and not ignore_err: if log.get_verbose(): log.error('Error from subprocess') if err is not None and err != '': print('err: ' + str(err), file=sys.stderr) if out is not None and out != '': print('out: ' + str(out), file=sys.stderr) raise CalledProcessError(proc.poll(), args, out, err) if log.get_verbose(): log.debug('Output of ' + repr(args)) if out is not None: print(out.decode()) if err is not None: print(err.decode()) if out is not None: return out.decode()
def populate_init(self): """ Populate the <proj>/__init__.py with meta info """ log.debug('Populating the __init__') init_path = join(self.get_name().lower(), '__init__.py') with open(init_path, 'r') as fh: init_file = fh.read() def replace_or_create(key, value): nonlocal init_file pattern = "^" + key + " ?= ?.*" # key at the beginning of the line preg = re.compile(pattern, re.M) if len(preg.findall(init_file)) == 0: init_file += '\n' + key + ' = "' + value + '"' log.debug(key + ' not found in init') else: init_file = preg.sub(key + ' = "' + value + '"', init_file) log.debug('Found ' + key + ' and replaced with ' + value) replace_or_create('__name__', self.meta['project_info']['name']) replace_or_create('__version__', self.meta['project_vcs']['version']) replace_or_create('__author__', self.meta['project_authors'][0]['name']) replace_or_create('__url__', self.meta['project_info']['url']) replace_or_create('__email__', self.meta['project_authors'][0]['email']) os.remove(init_path) with open(init_path, 'w+') as fh: fh.write(init_file) log.success('Populated __init__.py')
def call_check(args, ignore="", exclude=''): flakes = '' for dirname, _, files in os.walk(args): if os.path.ismount(dirname) or os.path.islink(dirname): log.warning('Ignored mounted/link directory: ' + dirname) continue if match_gitignore(dirname, exclude): continue for f in files: if not f.endswith('.py'): continue log.debug('Check: ' + f) log.set_additional_info(f) if match_gitignore(join(dirname, f), exclude): continue flak = call_with_stdout('python -m pyflakes ' + join(dirname, f), ignore_err=True) if flak is not None: flakes += flak # flakes = call_with_stdout('python -m pyflakes ' + args, ignore_err=True) pep8 = call_with_stdout('python -m pycodestyle --ignore=' + ignore + ' ' + ('' if exclude == '' else '--exclude=' + exclude) + ' ' + args, ignore_err=True) return flakes, pep8
def _retreive_version(): try: version = spvm.__version__ log.debug("SPVM version " + str(version)) lastver = ioutils.query_get("https://pypi.org/pypi/spvm/json")['info']['version'] log.debug('Last version is ' + lastver) if version != lastver: log.warning("A new version of spvm is available (" + lastver + ") you have version " + version) log.warning("Run pip install spvm --upgrade") except BaseException: log.warning('Could not get last version')
def replace_or_create(key, value): nonlocal init_file pattern = "^" + key + " ?= ?.*" # key at the beginning of the line preg = re.compile(pattern, re.M) if len(preg.findall(init_file)) == 0: init_file += '\n' + key + ' = "' + value + '"' log.debug(key + ' not found in init') else: init_file = preg.sub(key + ' = "' + value + '"', init_file) log.debug('Found ' + key + ' and replaced with ' + value)
def load_gitignore(self): if not os.path.isfile(join(self.location, '.gitignore')): self.gitignore = '' return with open(join(self.location, '.gitignore'), 'r') as fh: gi = fh.read() gi = gi.replace('\n\n', '\n').strip().split('\n') gi = [join(self.location, e) for e in gi] gi.append(join(self.location, '.git')) self.gitignore = ','.join(gi) log.debug('Ignoring: ' + self.gitignore)
def get_project_size(self): total_size = 0 for dirpath, _, filenames in os.walk(self.location): if self.match_gitignore(dirpath): log.debug('Ignored ' + dirpath) continue for f in filenames: fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) return sizeof_fmt(total_size)
def cli(verbose, mock, signed, repair, nocheck, update, notest, yes): log.set_verbose(verbose) log.debug('pwd: ' + os.getcwd()) cfg.config['mock'] = mock cfg.config['signed'] = signed cfg.config['repair'] = repair cfg.config['check'] = not nocheck cfg.config['update'] = update cfg.config['test'] = not notest cfg.config['ask'] = not yes if mock: log.warning('Mock Mode enabled') core.check_script_version()
def maybe_load_meta(self): """ If a metaFile exists, it will be loaded """ self.meta = None if os.path.isfile(self.projectMetaFile): try: with open(self.projectMetaFile, 'r') as fh: self.meta = json.loads(fh.read()) log.debug('Loaded project meta') except json.JSONDecodeError as e: log.error(repr(e)) log.error(config.metaFileName + " is invalid") exit(1) else: log.debug('No meta to load')
def _release_git(self, credentials = None): if os.path.isfile('.git-credentials'): os.remove('.git-credentials') log.success('Removed dangling credential file') # Commit version commit_message = self.meta['project_vcs']['release']['commit_template'].replace('%s', self.meta['project_vcs']['version']).replace('"', '\\"').strip() log.debug('Commit message: ' + commit_message) ioutils.call_git('add .') key = self.meta['project_vcs']['release']['git_signing_key'] if key != '': log.success(Fore.GREEN + config.PADLOCK + 'Commit will be signed with ' + key) ioutils.call_commit(commit_message, key=key) # Tag version tag = self.meta['project_vcs']['release']['tag_template'].replace('%s', self.meta['project_vcs']['version']) ioutils.call_git('tag ' + ('' if key == '' else '-u ' + key + ' ') + '-m ' + tag + ' ' + tag) log.success('Tagged: ' + tag) try: # Login if credentials != None: log.fine('Setting git credentials to temporary file') u = urllib.parse.urlparse(self.meta['project_vcs']['code_repository']) with open('.git-credentials', 'w+') as fh: fh.write(u.scheme+'://'+credentials['login']+':'+credentials['password']+'@'+u.hostname+'\n') ioutils.call_git(['config', 'credential.helper', 'store --file .git-credentials', '--replace-all']) log.success('Credentials are set') # Push repo = self.meta['project_vcs']['code_repository'] log.success('Pushing to ' + repo) ioutils.call_git('push ' + repo + ' --signed=if-asked') log.success('Pushing tags') ioutils.call_git('push ' + repo + ' --tags --signed=if-asked') finally: if credentials != None: os.remove('.git-credentials') log.success('Removed temporary credential file')
def up_version(self, kind): # FIXME other to 0 """ Increase the version in the project meta base on the 'kind' instruction: kind can be a str or a number if a number, it is treated like an index for the version increment if a string, it can be major, minor or patch """ log.fine('Increasing version (' + str(kind) + ')') v = self.meta['project_vcs']['version'] v_ = [int(i) for i in v.split('.')] while len(v_) < 3: v_.insert(0, '0') log.debug("Current version comphrension: " + str(v_)) if kind.isdigit(): kind = int(kind) if kind < 0 or kind >= len(v_): log.error('Unrecognized version changer: ' + str(kind)) v_[int(kind)] += 1 else: kind = kind.lower() if kind == 'patch': index = len(v_) - 1 elif kind == 'major': index = 0 elif kind == 'minor': index = len(v_) - 2 elif kind == 'pass': log.success('Version not changed') return else: log.error('Unrecognized version changer: ' + str(kind)) exit(1) v_[index] += 1 v_ = [v_[i] if i <= index else 0 for i in range(len(v_))] self.meta['project_vcs']['version'] = '.'.join([str(i) for i in v_]) self.save_project_info() log.success(v + ' -> ' + self.get_version())
def __init__(self, location=os.getcwd()): self.location = os.path.normpath(location) # directory for project if os.path.isdir(self.location): os.chdir(self.location) log.debug("cwd switched: " + self.location) log.debug("New project instance @ " + self.location) log.debug("Project status is: " + str(self.get_project_status())) self.projectMetaFile = join(self.location, config.metaFileName) self.maybe_load_meta() if self.meta is not None: metautils.check_project_meta(self.meta) self.save_project_info() self.load_gitignore()
def _release_docker(self, credentials = None): log.success('Building Docker Image') client = docker.from_env() log.debug(json.dumps(client.version(), indent=4)) status = {} def _show_docker_progress(obj): nonlocal status if 'errorDetail' in obj: log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET) raise docker.errors.DockerException(obj['errorDetail']['message']) if 'stream' in obj: for line in obj['stream'].split('\n'): if line == '': continue log.success(line.strip()) status.clear() return if 'status' in obj: if 'id' not in obj: log.success(obj['status']) return if len(status) == 0: print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET) s = obj['id'].strip() + ' ' + obj['status'] + '\t' if 'progress' in obj: s += obj['progress'] if obj['id'] not in status: status[obj['id']] = {'index': len(status) + 1, 'str': s} print(s) return status[obj['id']]['str'] = s print('\033[F' * (len(status) + 1)) for e in status: print('\033[K' + status[e]['str']) # print('\n'*i, end = '') # FIXME choose dockerfile rep = self.meta['project_vcs']['docker_repository'] log.success('Image repo: ' + rep) if hasattr(client, 'api'): client = client.api g = client.build(tag=rep, path='.', dockerfile='Dockerfile') for line in g: _show_docker_progress(json.loads(line.decode())) if config.config['mock']: log.warning(Fore.YELLOW + 'Mock mode: not pushing' + Fore.RESET) return if credentials != None: try: client.login(credentials['login'], credentials['password']) except docker.errors.APIError as excep: log.error('Cannot login: '******'Logged in as '+credentials['login']) log.success('Pushing image') for line in client.push(rep, stream=True): _show_docker_progress(json.loads(line.decode()))
def run(scriptname, projectname): """ Run a script defined in pyp.json """ log.debug('Script from CLI: ' + scriptname) get_project(projectname).run(scriptname)
def init(projectname): """ Initializes a spvm project, use init projectname to create sub-directory projectname""" log.debug('Running init') get_project(projectname).init()
def check_packages(base_url='https://pypi.python.org/pypi/'): log.fine('Checking packages in: ' + piptmp) unchecked = 0 for f in os.listdir(piptmp): try: log.set_additional_info(f) f_ = piptmp + os.sep + f if not os.path.isfile(f_): continue splited = f.split('-') log.debug('Checking ' + splited[0]) package_info = query_get(base_url + splited[0] + '/' + splited[1] + '/json') for f_info in package_info['releases'][splited[1]]: if not os.path.isfile( os.path.join(piptmp, f_info['filename'])): continue if md5(f_) != f_info['md5_digest']: log.error('Hash do not match') exit(1) # log.success(Fore.GREEN+'Hash checked for '+f) if not f_info['has_sig']: log.debug(Fore.YELLOW + 'No signature provided for ' + f_info['filename']) # FIXME throw? unchecked += 1 continue sig = query_get(f_info['url'] + '.asc', False) log.debug('File: ' + f_info['filename'] + ' has signature:\n ' + sig.decode()) # Check q = '' if log.get_verbose() else ' --quiet' try: call_gpg('--no-default-keyring --keyring tmp.gpg' + q + ' --auto-key-retrieve --verify - ' + f_, inp=sig) # FIXME Only use known keys? except CalledProcessError as er: if er.returncode == 1: log.error(Fore.RED + config.OPEN_PADLOCK + ' Invalid signature for ' + f) exit(1) log.error('Could not check signature for ' + f + ' (' + repr(er) + ')') unchecked += 1 continue log.success(Fore.GREEN + config.PADLOCK + ' File ' + f + ' is verified') except KeyboardInterrupt: exit(2) except SystemExit as e: raise e except BaseException as be: log.error(Fore.RED + config.OPEN_PADLOCK + ' Failed to check ' + f + Fore.RESET) log.error(repr(be)) log.warning(Fore.YELLOW + str(unchecked) + ' file(s) could not be verified')