def save_project_info(self):
"""
Write the current project info to the metaFile
"""
with open(self.projectMetaFile, 'w+') as fh:
fh.write(json.dumps(self.meta, indent=4))
log.debug('Saved project meta')
def call_with_stdout(args,
ignore_err=False,
stdout=PIPE,
inp=None,
stderr=PIPE):
with Popen(args.split(' ') if type(args) == str else args,
stdout=stdout,
stdin=PIPE if inp is not None else None,
stderr=stderr) as proc:
out, err = proc.communicate(input=inp)
if proc.poll() != 0 and not ignore_err:
if log.get_verbose():
log.error('Error from subprocess')
if err is not None and err != '':
print('err: ' + str(err), file=sys.stderr)
if out is not None and out != '':
print('out: ' + str(out), file=sys.stderr)
raise CalledProcessError(proc.poll(), args, out, err)
if log.get_verbose():
log.debug('Output of ' + repr(args))
if out is not None:
print(out.decode())
if err is not None:
print(err.decode())
if out is not None:
return out.decode()
def populate_init(self):
""" Populate the <proj>/__init__.py with meta info """
log.debug('Populating the __init__')
init_path = join(self.get_name().lower(), '__init__.py')
with open(init_path, 'r') as fh:
init_file = fh.read()
def replace_or_create(key, value):
nonlocal init_file
pattern = "^" + key + " ?= ?.*" # key at the beginning of the line
preg = re.compile(pattern, re.M)
if len(preg.findall(init_file)) == 0:
init_file += '\n' + key + ' = "' + value + '"'
log.debug(key + ' not found in init')
else:
init_file = preg.sub(key + ' = "' + value + '"', init_file)
log.debug('Found ' + key + ' and replaced with ' + value)
replace_or_create('__name__', self.meta['project_info']['name'])
replace_or_create('__version__', self.meta['project_vcs']['version'])
replace_or_create('__author__', self.meta['project_authors'][0]['name'])
replace_or_create('__url__', self.meta['project_info']['url'])
replace_or_create('__email__', self.meta['project_authors'][0]['email'])
os.remove(init_path)
with open(init_path, 'w+') as fh:
fh.write(init_file)
log.success('Populated __init__.py')
def call_check(args, ignore="", exclude=''):
flakes = ''
for dirname, _, files in os.walk(args):
if os.path.ismount(dirname) or os.path.islink(dirname):
log.warning('Ignored mounted/link directory: ' + dirname)
continue
if match_gitignore(dirname, exclude):
continue
for f in files:
if not f.endswith('.py'):
continue
log.debug('Check: ' + f)
log.set_additional_info(f)
if match_gitignore(join(dirname, f), exclude):
continue
flak = call_with_stdout('python -m pyflakes ' + join(dirname, f),
ignore_err=True)
if flak is not None:
flakes += flak
# flakes = call_with_stdout('python -m pyflakes ' + args, ignore_err=True)
pep8 = call_with_stdout('python -m pycodestyle --ignore=' + ignore + ' ' +
('' if exclude == '' else '--exclude=' + exclude) +
' ' + args,
ignore_err=True)
return flakes, pep8
def _retreive_version():
try:
version = spvm.__version__
log.debug("SPVM version " + str(version))
lastver = ioutils.query_get("https://pypi.org/pypi/spvm/json")['info']['version']
log.debug('Last version is ' + lastver)
if version != lastver:
log.warning("A new version of spvm is available (" + lastver + ") you have version " + version)
log.warning("Run pip install spvm --upgrade")
except BaseException:
log.warning('Could not get last version')
def replace_or_create(key, value):
nonlocal init_file
pattern = "^" + key + " ?= ?.*" # key at the beginning of the line
preg = re.compile(pattern, re.M)
if len(preg.findall(init_file)) == 0:
init_file += '\n' + key + ' = "' + value + '"'
log.debug(key + ' not found in init')
else:
init_file = preg.sub(key + ' = "' + value + '"', init_file)
log.debug('Found ' + key + ' and replaced with ' + value)
def load_gitignore(self):
if not os.path.isfile(join(self.location, '.gitignore')):
self.gitignore = ''
return
with open(join(self.location, '.gitignore'), 'r') as fh:
gi = fh.read()
gi = gi.replace('\n\n', '\n').strip().split('\n')
gi = [join(self.location, e) for e in gi]
gi.append(join(self.location, '.git'))
self.gitignore = ','.join(gi)
log.debug('Ignoring: ' + self.gitignore)
def get_project_size(self):
total_size = 0
for dirpath, _, filenames in os.walk(self.location):
if self.match_gitignore(dirpath):
log.debug('Ignored ' + dirpath)
continue
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return sizeof_fmt(total_size)
def cli(verbose, mock, signed, repair, nocheck, update, notest, yes):
log.set_verbose(verbose)
log.debug('pwd: ' + os.getcwd())
cfg.config['mock'] = mock
cfg.config['signed'] = signed
cfg.config['repair'] = repair
cfg.config['check'] = not nocheck
cfg.config['update'] = update
cfg.config['test'] = not notest
cfg.config['ask'] = not yes
if mock:
log.warning('Mock Mode enabled')
core.check_script_version()
def maybe_load_meta(self):
"""
If a metaFile exists, it will be loaded
"""
self.meta = None
if os.path.isfile(self.projectMetaFile):
try:
with open(self.projectMetaFile, 'r') as fh:
self.meta = json.loads(fh.read())
log.debug('Loaded project meta')
except json.JSONDecodeError as e:
log.error(repr(e))
log.error(config.metaFileName + " is invalid")
exit(1)
else:
log.debug('No meta to load')
def _release_git(self, credentials = None):
if os.path.isfile('.git-credentials'):
os.remove('.git-credentials')
log.success('Removed dangling credential file')
# Commit version
commit_message = self.meta['project_vcs']['release']['commit_template'].replace('%s', self.meta['project_vcs']['version']).replace('"', '\\"').strip()
log.debug('Commit message: ' + commit_message)
ioutils.call_git('add .')
key = self.meta['project_vcs']['release']['git_signing_key']
if key != '':
log.success(Fore.GREEN + config.PADLOCK + 'Commit will be signed with ' + key)
ioutils.call_commit(commit_message, key=key)
# Tag version
tag = self.meta['project_vcs']['release']['tag_template'].replace('%s', self.meta['project_vcs']['version'])
ioutils.call_git('tag ' + ('' if key == '' else '-u ' + key + ' ') + '-m ' + tag + ' ' + tag)
log.success('Tagged: ' + tag)
try:
# Login
if credentials != None:
log.fine('Setting git credentials to temporary file')
u = urllib.parse.urlparse(self.meta['project_vcs']['code_repository'])
with open('.git-credentials', 'w+') as fh:
fh.write(u.scheme+'://'+credentials['login']+':'+credentials['password']+'@'+u.hostname+'\n')
ioutils.call_git(['config', 'credential.helper', 'store --file .git-credentials', '--replace-all'])
log.success('Credentials are set')
# Push
repo = self.meta['project_vcs']['code_repository']
log.success('Pushing to ' + repo)
ioutils.call_git('push ' + repo + ' --signed=if-asked')
log.success('Pushing tags')
ioutils.call_git('push ' + repo + ' --tags --signed=if-asked')
finally:
if credentials != None:
os.remove('.git-credentials')
log.success('Removed temporary credential file')
def up_version(self, kind): # FIXME other to 0
"""
Increase the version in the project meta base on the 'kind' instruction:
kind can be a str or a number
if a number, it is treated like an index for the version increment
if a string, it can be major, minor or patch
"""
log.fine('Increasing version (' + str(kind) + ')')
v = self.meta['project_vcs']['version']
v_ = [int(i) for i in v.split('.')]
while len(v_) < 3:
v_.insert(0, '0')
log.debug("Current version comphrension: " + str(v_))
if kind.isdigit():
kind = int(kind)
if kind < 0 or kind >= len(v_):
log.error('Unrecognized version changer: ' + str(kind))
v_[int(kind)] += 1
else:
kind = kind.lower()
if kind == 'patch':
index = len(v_) - 1
elif kind == 'major':
index = 0
elif kind == 'minor':
index = len(v_) - 2
elif kind == 'pass':
log.success('Version not changed')
return
else:
log.error('Unrecognized version changer: ' + str(kind))
exit(1)
v_[index] += 1
v_ = [v_[i] if i <= index else 0 for i in range(len(v_))]
self.meta['project_vcs']['version'] = '.'.join([str(i) for i in v_])
self.save_project_info()
log.success(v + ' -> ' + self.get_version())
def __init__(self, location=os.getcwd()):
self.location = os.path.normpath(location) # directory for project
if os.path.isdir(self.location):
os.chdir(self.location)
log.debug("cwd switched: " + self.location)
log.debug("New project instance @ " + self.location)
log.debug("Project status is: " + str(self.get_project_status()))
self.projectMetaFile = join(self.location, config.metaFileName)
self.maybe_load_meta()
if self.meta is not None:
metautils.check_project_meta(self.meta)
self.save_project_info()
self.load_gitignore()
def _release_docker(self, credentials = None):
log.success('Building Docker Image')
client = docker.from_env()
log.debug(json.dumps(client.version(), indent=4))
status = {}
def _show_docker_progress(obj):
nonlocal status
if 'errorDetail' in obj:
log.error(Fore.RED + 'Error: ' + str(obj['errorDetail']['message']) + Fore.RESET)
raise docker.errors.DockerException(obj['errorDetail']['message'])
if 'stream' in obj:
for line in obj['stream'].split('\n'):
if line == '':
continue
log.success(line.strip())
status.clear()
return
if 'status' in obj:
if 'id' not in obj:
log.success(obj['status'])
return
if len(status) == 0:
print(Fore.GREEN + "\rA docker I/O operation is in progress" + Fore.RESET)
s = obj['id'].strip() + ' ' + obj['status'] + '\t'
if 'progress' in obj:
s += obj['progress']
if obj['id'] not in status:
status[obj['id']] = {'index': len(status) + 1, 'str': s}
print(s)
return
status[obj['id']]['str'] = s
print('\033[F' * (len(status) + 1))
for e in status:
print('\033[K' + status[e]['str'])
# print('\n'*i, end = '')
# FIXME choose dockerfile
rep = self.meta['project_vcs']['docker_repository']
log.success('Image repo: ' + rep)
if hasattr(client, 'api'):
client = client.api
g = client.build(tag=rep, path='.', dockerfile='Dockerfile')
for line in g:
_show_docker_progress(json.loads(line.decode()))
if config.config['mock']:
log.warning(Fore.YELLOW + 'Mock mode: not pushing' + Fore.RESET)
return
if credentials != None:
try:
client.login(credentials['login'], credentials['password'])
except docker.errors.APIError as excep:
log.error('Cannot login: '******'Logged in as '+credentials['login'])
log.success('Pushing image')
for line in client.push(rep, stream=True):
_show_docker_progress(json.loads(line.decode()))
def run(scriptname, projectname):
""" Run a script defined in pyp.json """
log.debug('Script from CLI: ' + scriptname)
get_project(projectname).run(scriptname)
def init(projectname):
""" Initializes a spvm project, use init projectname to create sub-directory projectname"""
log.debug('Running init')
get_project(projectname).init()
def check_packages(base_url='https://pypi.python.org/pypi/'):
log.fine('Checking packages in: ' + piptmp)
unchecked = 0
for f in os.listdir(piptmp):
try:
log.set_additional_info(f)
f_ = piptmp + os.sep + f
if not os.path.isfile(f_):
continue
splited = f.split('-')
log.debug('Checking ' + splited[0])
package_info = query_get(base_url + splited[0] + '/' +
splited[1] + '/json')
for f_info in package_info['releases'][splited[1]]:
if not os.path.isfile(
os.path.join(piptmp, f_info['filename'])):
continue
if md5(f_) != f_info['md5_digest']:
log.error('Hash do not match')
exit(1)
# log.success(Fore.GREEN+'Hash checked for '+f)
if not f_info['has_sig']:
log.debug(Fore.YELLOW + 'No signature provided for ' +
f_info['filename']) # FIXME throw?
unchecked += 1
continue
sig = query_get(f_info['url'] + '.asc', False)
log.debug('File: ' + f_info['filename'] +
' has signature:\n ' + sig.decode())
# Check
q = '' if log.get_verbose() else ' --quiet'
try:
call_gpg('--no-default-keyring --keyring tmp.gpg' + q +
' --auto-key-retrieve --verify - ' + f_,
inp=sig) # FIXME Only use known keys?
except CalledProcessError as er:
if er.returncode == 1:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Invalid signature for ' + f)
exit(1)
log.error('Could not check signature for ' + f + ' (' +
repr(er) + ')')
unchecked += 1
continue
log.success(Fore.GREEN + config.PADLOCK + ' File ' + f +
' is verified')
except KeyboardInterrupt:
exit(2)
except SystemExit as e:
raise e
except BaseException as be:
log.error(Fore.RED + config.OPEN_PADLOCK +
' Failed to check ' + f + Fore.RESET)
log.error(repr(be))
log.warning(Fore.YELLOW + str(unchecked) +
' file(s) could not be verified')
