def process(self, verbose=False, warning_as_error=False):
if not os.path.isfile(os.path.join(self.cwd, self.filename)):
raise PyvenException('Test file not found : ' +
os.path.join(self.path, self.filename))
if os.path.isdir(self.cwd):
Logger.get().info('Running test : ' + self.filename)
if os.path.isfile(
os.path.join(self.cwd, self._format_report_name())):
os.remove(os.path.join(self.cwd, self._format_report_name()))
self.duration, out, err, returncode = self._call_command(
self._format_call())
self.parser.parse(
os.path.join(self.cwd, self._format_report_name()))
if returncode != 0:
self.status = pyven.constants.STATUS[1]
if os.path.isfile(
os.path.join(self.cwd, self._format_report_name())):
self.errors = self.parser.errors
else:
msg = 'Could not find TRX report --> ' + os.path.join(
self.path, self._format_report_name())
self.errors.append([msg])
Logger.get().error(msg)
Logger.get().error('Test failed : ' + self.filename)
else:
self.status = pyven.constants.STATUS[0]
Logger.get().info('Test OK : ' + self.filename)
return returncode == 0
Logger.get().error('Unknown directory : ' + self.path)
return False
def process(self, verbose=False, warning_as_error=False):
Logger.get().info('Building : ' + self.type + ':' + self.name)
self.duration, out, err, returncode = self._call_command(
self._format_call())
if verbose:
for line in out.splitlines():
Logger.get().info('[' + self.type + ']' + line)
for line in err.splitlines():
Logger.get().info('[' + self.type + ']' + line)
self.parser.parse(out.splitlines())
warnings = self.parser.warnings
for w in warnings:
self.warnings.append([w[0].replace(w[0].split()[-1], '')])
if returncode != 0:
self.status = pyven.constants.STATUS[1]
errors = self.parser.errors
for e in errors:
if e[0].split()[-1].startswith(
'[') and e[0].split()[-1].endswith(']'):
self.errors.append([e[0].replace(e[0].split()[-1], '')])
else:
self.errors.append([e[0]])
Logger.get().error('Build failed : ' + self.type + ':' + self.name)
elif warning_as_error and len(self.warnings) > 0:
self.status = pyven.constants.STATUS[1]
Logger.get().error('Build failed : ' + self.type + ':' + self.name)
else:
self.status = pyven.constants.STATUS[0]
return returncode == 0 and (not warning_as_error
or len(self.warnings) == 0)
def _process(self, project):
ok = True
Logger.get().info('Delivering to directory ' + self.location)
packages = [p for p in project.packages.values()]
for repo in [
project.repositories[p.repo] for p in packages if p.to_retrieve
]:
if not repo.is_reachable():
msg = 'Repository ' + repo.name + ' --> unreachable for delivery'
Logger.get().error(msg)
self.checker.errors.append([msg])
ok = False
if ok:
for package in packages:
if package.to_retrieve:
package.deliver(self.location,
project.repositories[package.repo])
else:
package.deliver(self.location, Step.WORKSPACE)
Logger.get().info('Delivered package : ' +
package.format_name())
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def _format_call(self, clean=False):
call = [
'MSTest', '/resultsfile:' + self._format_report_name(),
'/testcontainer:' + self.filename
]
for argument in self.arguments:
call.append(argument)
Logger.get().info(' '.join(call))
return call
def check_generation(self, artifacts_checker):
if not os.path.isfile(self.file):
msg = ['Artifact not found : ' + self.format_name(),\
'Expected location : ' + self.file]
artifacts_checker.errors.append(msg)
Logger.get().error(msg[0])
Logger.get().error(msg[1])
return False
return True
def _format_call(self, clean=False):
if os.name == 'nt':
call = [self.filename]
call.append(self._format_report_name())
elif os.name == 'posix':
call = ['./' + self.filename + ' ' + self._format_report_name()]
for argument in self.arguments:
call.append(argument)
Logger.get().info(' '.join(call))
return call
def __intern(self, project, parser):
ok = True
try:
function(self, project, parser)
except PyvenException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
ok = False
return ok
def _format_call(self, clean=False):
call = ['make']
if clean:
call.append('clean')
else:
for option in self.options:
call.append(option)
for rule in self.rules:
call.append(rule)
Logger.get().info(' '.join(call))
return call
def _format_call(self):
call = [self.type, '-H.', '-B' + self.output_path, '-G']
if pyven.constants.PLATFORM == 'windows':
call.append(self.target_generator)
elif pyven.constants.PLATFORM == 'linux':
call.append('"' + self.target_generator + '"')
for definition in self.definitions:
call.append('-D' + definition)
if pyven.constants.PLATFORM == 'linux':
call = [' '.join(call)]
Logger.get().info(' '.join(call))
return call
def _intern(self, verbose=False, warning_as_error=False):
ok = True
try:
tic = time.time()
ok = function(self, verbose, warning_as_error)
toc = time.time()
Logger.get().info('Process time : ' +
str(round(toc - tic, 3)) + ' seconds')
except PyvenException as e:
Logger.get().error(e.args)
self.errors.append(e.args)
self.status = cst.STATUS[1]
ok = False
return ok
def _configure_projects(self, project, parser):
directories = parser.parse_projects()
for directory in directories:
if not os.path.isdir(os.path.join(project.path, directory)):
raise PyvenException('Subproject directory does not exist : ' +
directory)
else:
full_path = os.path.join(project.path, directory)
subproject = Project(full_path,
parser.plugins_manager.plugins.copy())
subproject.constants = project.constants.copy()
if not self._process(subproject):
raise PyvenException('Subproject ' + full_path +
' --> configuration failed')
Logger.get().info('Added subproject --> ' + directory)
def _process(self, project):
ok = True
for tool in project.builders:
if not tool.clean(self.verbose):
ok = False
for tool in project.preprocessors:
if not tool.clean(self.verbose):
ok = False
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def _configure_packages(self, project, parser):
packages = parser.parse_packages()
for package in packages:
package.company = project.replace_constants(package.company)
package.name = project.replace_constants(package.name)
package.config = project.replace_constants(package.config)
package.version = project.replace_constants(package.version)
if len(package.deliveries) > 0:
for i in range(len(package.deliveries)):
package.deliveries[i] = project.replace_constants(
package.deliveries[i])
items = []
items.extend(package.items)
package.items = []
if package.format_name() in project.packages.keys():
raise PyvenException('Package already added --> ' +
package.format_name())
elif package.to_retrieve and package.repo not in project.repositories.keys(
) and package.repo not in [Step.LOCAL_REPO.name, 'workspace']:
raise PyvenException('Package repository not declared --> ' +
package.format_name() + ' : repo ' +
package.repo)
else:
for item in items:
item = project.replace_constants(item)
if item not in project.artifacts.keys():
raise PyvenException('Package ' +
package.format_name() +
' : Artifact not declared --> ' +
item)
else:
package.items.append(project.artifacts[item])
Logger.get().info('Package ' + package.format_name() +
' : Artifact added --> ' + item)
project.packages[package.format_name()] = package
Logger.get().info('Package added --> ' + package.format_name())
if not package.publish:
Logger.get().info('Package ' + package.format_name() +
' --> publishment disabled')
for package in packages:
for extension in package.extensions:
extension = project.replace_constants(extension)
if extension not in project.packages.keys():
raise PyvenException(
'Package ' + package.format_name() +
' : Package extension not declared --> ' + extension)
elif project.packages[extension].to_retrieve:
raise PyvenException(
'Package ' + package.format_name() +
' : Cannot extend a package to be retrieved --> ' +
extension)
else:
package.items.extend(project.packages[extension].items)
Logger.get().info('Package ' + package.format_name() +
' : Package extension added --> ' +
extension)
def process(self):
self.parallelizer.threads = []
for project in Step.PROJECTS:
if len(project.unit_tests) == 0:
Logger.get().warning('No unit tests configured')
else:
for test in project.unit_tests:
self.parallelizer.threads.append(Thread(target=test.process, args=(self.verbose,)))
self.parallelizer.run()
ok = True
for project in Step.PROJECTS:
project.status = pyven.constants.STATUS[0]
for test in project.unit_tests:
if test.status in pyven.constants.STATUS[1:]:
project.status = pyven.constants.STATUS[1]
ok = False
return ok
def _process(self, project):
Logger.get().info('Starting ' + self.name)
ok = True
for artifact in [a for a in project.artifacts.values() if not a.to_retrieve]:
if not artifact.check(self.checker):
Logger.get().error('Artifact ' + artifact.format_name() + ' --> KO')
ok = False
else:
Logger.get().info('Artifact ' + artifact.format_name() + ' --> OK')
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error('Artifacts missing')
else:
for artifact in [a for a in project.artifacts.values() if not a.to_retrieve]:
Step.WORKSPACE.publish(artifact, artifact.file)
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def clean(self, verbose=False, warning_as_error=False):
Logger.get().info('Cleaning : ' + self.type + ':' + self.name)
if os.path.isfile(os.path.join(self.cwd, self.project)):
self.duration, out, err, returncode = self._call_command(
self._format_call(clean=True))
if verbose:
for line in out.splitlines():
Logger.get().info('[' + self.type + ']' + line)
for line in err.splitlines():
Logger.get().info('[' + self.type + ']' + line)
if returncode != 0:
Logger.get().error('Clean failed : ' + self.type + ':' +
self.name)
return returncode == 0
Logger.get().info('No project to be cleaned : ' + self.project)
self.status = pyven.constants.STATUS[0]
return True
def _intern(self, project):
ok = True
Logger.set_format(project)
try:
try:
if project.path != os.getcwd():
if not os.path.isdir(project.path):
raise PyvenException(
'Subproject path does not exist : ' +
project.path)
tic = time.time()
ok = function(self, project)
toc = time.time()
Logger.get().info('Step time : ' +
str(round(toc - tic, 3)) + ' seconds')
except PyvenException as e:
self.checker.errors.append(e.args)
ok = False
finally:
Logger.set_format()
return ok
def _format_call(self, clean=False):
exe = 'msbuild.exe'
if self.dot_net_version is not None:
Logger.get().info('Using .NET ' + self.dot_net_version)
exe = os.path.join(os.environ.get('WINDIR'), 'Microsoft.NET',
'Framework', self.dot_net_version, exe)
elif self.project.endswith('.dproj'):
Logger.get().info('Using .NET v3.5')
exe = os.path.join(os.environ.get('WINDIR'), 'Microsoft.NET',
'Framework', 'v3.5', exe)
call = [exe, self.project]
call.append(
'/consoleLoggerParameters:NoSummary;ErrorsOnly;WarningsOnly')
if self.project.endswith('.sln'):
call.append('/property:Configuration=' + self.configuration)
call.append('/property:Platform=' + self.architecture)
if clean:
call.append('/t:clean')
elif self.project.endswith('.dproj'):
call.append('/p:config=' + self.configuration)
call.append('/p:platform=' + self.architecture)
if clean:
call.append('/t:clean')
else:
call.append('/t:build')
else:
raise PyvenException(
'Project format not supported : ' + self.project,
'Supported formats : *.sln, *.dproj')
for option in self.options:
call.append(option)
Logger.get().info(' '.join(call))
return call
def _process(self, project):
Logger.get().info('Starting ' + self.name)
ok = True
for tool in project.postprocessors:
tic = time.time()
if not tool.process(self.verbose):
ok = False
else:
toc = time.time()
Logger.get().info('Time for ' + tool.type + ':' + tool.name + ' : ' + str(round(toc - tic, 3)) + ' seconds')
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def _configure_artifacts(self, project, parser):
artifacts = parser.parse_artifacts()
for artifact in artifacts:
artifact.company = project.replace_constants(artifact.company)
artifact.name = project.replace_constants(artifact.name)
artifact.config = project.replace_constants(artifact.config)
artifact.version = project.replace_constants(artifact.version)
if not artifact.to_retrieve:
artifact.file = project.replace_constants(artifact.file)
if artifact.format_name() in project.artifacts.keys():
raise PyvenException('Artifact already added --> ' +
artifact.format_name())
elif artifact.to_retrieve and artifact.repo not in project.repositories.keys(
) and artifact.repo not in [Step.LOCAL_REPO.name, 'workspace']:
raise PyvenException('Artifact repository not declared --> ' +
artifact.format_name() + ' : repo ' +
artifact.repo)
else:
project.artifacts[artifact.format_name()] = artifact
Logger.get().info('Artifact added --> ' +
artifact.format_name())
if not artifact.publish:
Logger.get().info('Artifact ' + artifact.format_name() +
' --> publishment disabled')
def _process(self, project):
ok = True
for artifact in [a for a in project.artifacts.values() if a.publish]:
Step.LOCAL_REPO.publish(artifact, Step.WORKSPACE)
Logger.get().info('Repository ' + Step.LOCAL_REPO.name +
' --> Published artifact ' +
artifact.format_name())
for package in [p for p in project.packages.values() if p.publish]:
Step.LOCAL_REPO.publish(package, Step.WORKSPACE)
Logger.get().info('Repository ' + Step.LOCAL_REPO.name +
' --> Published package ' +
package.format_name())
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def process(self, verbose=False, warning_as_error=False):
Logger.get().info('CMake : ' + self.type + ':' + self.name)
self.duration, out, err, returncode = self._call_command(
self._format_call())
if verbose:
for line in out.splitlines():
Logger.get().info('[' + self.type + ']' + line)
for line in err.splitlines():
Logger.get().info('[' + self.type + ']' + line)
self.parser.parse(out.splitlines() + err.splitlines())
self.warnings = self.parser.warnings
if returncode != 0:
self.status = pyven.constants.STATUS[1]
self.errors = self.parser.errors
Logger.get().error('CMake failed : ' + self.type + ':' + self.name)
else:
self.status = pyven.constants.STATUS[0]
return returncode == 0
def retrieve(type, project, items, checker):
ok = True
for item in [i for i in items.values() if i.to_retrieve and i.repo]:
try:
if project.repositories[item.repo].is_reachable():
if not project.repositories[item.repo].is_available(item):
raise RepositoryException('Repository ' + item.repo +
' --> ' + type + ' ' +
item.format_name() +
' not available')
elif item.repo == Step.WORKSPACE.name:
item.file = os.path.join(
item.location(Step.WORKSPACE.url),
os.listdir(item.location(Step.WORKSPACE.url))[0])
Logger.get().info('Workspace --> Retrieved ' + type +
' : ' + item.format_name())
else:
project.repositories[item.repo].retrieve(
item, Step.WORKSPACE)
Logger.get().info('Repository ' + item.repo +
' --> Retrieved ' + type + ' : ' +
item.format_name())
elif Step.WORKSPACE.is_available(item):
item.file = os.path.join(
item.location(Step.WORKSPACE.url),
os.listdir(item.location(Step.WORKSPACE.url))[0])
Logger.get().info('Workspace --> Retrieved ' + type + ' : ' +
item.format_name())
else:
raise RepositoryException(
'Repository ' + item.repo +
' unreachable --> Unable to retrieve ' + type + ' : ' +
item.format_name())
except RepositoryException as e:
checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
ok = False
return ok
def _intern(self):
Step.log_delimiter()
Logger.get().info('STEP ' + self.name.replace('_', ' ').upper() +
' : STARTING')
ok = function(self)
if ok:
self.status = pyven.constants.STATUS[0]
Logger.get().info('STEP ' +
self.name.replace('_', ' ').upper() +
' : SUCCESSFUL')
Step.log_delimiter()
else:
self.status = pyven.constants.STATUS[1]
Logger.get().info('STEP ' +
self.name.replace('_', ' ').upper() +
' : FAILED')
Step.log_delimiter()
return ok
def _process(self, project):
ok = retrieve('artifact', project,
project.artifacts, self.checker) and retrieve(
'package', project, project.packages, self.checker)
if ok:
for package in [
p for p in project.packages.values() if not p.to_retrieve
]:
try:
if not package.pack(Step.WORKSPACE):
ok = False
except PyvenException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
ok = False
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def _configure_repositories(self, project, parser):
repositories = parser.parse_repositories()
project.repositories[Step.WORKSPACE.name] = Step.WORKSPACE
project.repositories[Step.LOCAL_REPO.name] = Step.LOCAL_REPO
for repo in repositories:
if repo.name == 'workspace' or repo.name == Step.LOCAL_REPO.name:
raise PyvenException('Repository name reserved --> ' +
repo.name + ' : ' + repo.url)
else:
if repo.name in project.repositories.keys():
raise PyvenException('Repository already added --> ' +
repo.name + ' : ' + repo.url)
else:
project.repositories[repo.name] = repo
if repo.is_reachable():
if repo.release:
Logger.get().info('Release repository added --> ' +
repo.name + ' : ' + repo.url)
else:
Logger.get().info('Repository added --> ' +
repo.name + ' : ' + repo.url)
else:
Logger.get().warning('Repository not accessible --> ' +
repo.name + ' : ' + repo.url)
def _process(self, project):
ok = True
repositories = [r for r in project.repositories.values() if (not r.release or (r.release and self.release))\
and r.name != Step.WORKSPACE.name\
and r.name != Step.LOCAL_REPO.name]
for repo in repositories:
if not repo.is_reachable():
msg = 'Repository ' + repo.name + ' --> unreachable for deployment'
Logger.get().error(msg)
self.checker.errors.append([msg])
ok = False
if ok:
for repo in repositories:
try:
for artifact in [
a for a in project.artifacts.values() if a.publish
]:
repo.publish(artifact, Step.WORKSPACE, self.overwrite)
Logger.get().info('Repository ' + repo.name +
' --> Published artifact ' +
artifact.format_name())
for package in [
p for p in project.packages.values() if p.publish
]:
repo.publish(package, Step.WORKSPACE, self.overwrite)
Logger.get().info('Repository ' + repo.name +
' --> Published package ' +
package.format_name())
except RepositoryException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
raise e
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def clean(self, verbose=False, warning_as_error=False):
Logger.get().info('Cleaning : ' + self.type + ':' + self.name)
if os.path.isdir(os.path.join(self.cwd, self.output_path)):
shutil.rmtree(os.path.join(self.cwd, self.output_path))
self.status = pyven.constants.STATUS[0]
return True
def log_delimiter(path=None):
Logger.get().info('===================================')
def main(args):
tic = time.time()
Logger.get().info('Pyven ' + cst.VERSION)
parser = argparse.ArgumentParser()
parser.add_argument('--version', action='version', version='1.0.0')
parser.add_argument(
'--display',
'-d',
action='store_true',
help='display build report in the webbrowser right after build')
parser.add_argument('--verbose',
'-v',
action='store_true',
help='increase verbosity level')
parser.add_argument('--warning-as-error',
'-wae',
dest='warning_as_error',
action='store_true',
help='consider warnings as errors')
parser.add_argument(
'--lines',
'-l',
dest='nb_lines',
action='store',
type=int,
default=10,
help='Number of errors/warnings to be displayed in the build report')
parser.add_argument('--custom-pym',
'-cp',
dest='pym',
action='store',
type=str,
default='pym.xml',
help='pym file name')
parser.add_argument('--release',
'-r',
action='store_true',
help='enable deployment to release repositories')
parser.add_argument('--overwrite',
'-o',
action='store_true',
help='enable deployment to release repositories')
parser.add_argument('--report-style',
'-rs',
dest='report_style',
action='store',
type=str,
default='default',
help='Sets the HTML report style')
parser.add_argument('--multithread',
'-m',
dest='nb_threads',
action='store',
type=int,
default=1,
help='Number of threads for parallel build')
parser.add_argument('step',
choices=Pyven.STEPS + Pyven.UTILS,
help='pyven step to be achieved')
parser.add_argument(
'path',
nargs='?',
help='path to the delivery directory (used with "deliver" step only)')
args = parser.parse_args()
if args.step not in ['deliver', 'parse'] and args.path is not None:
parser.error('Too many arguments provided')
if args.step in ['deliver', 'parse'] and args.path is None:
parser.error('Missing path argument for step ' + args.step)
pvn = Pyven(args.step,
args.verbose,
args.warning_as_error,
args.pym,
args.release,
args.overwrite,
arguments={'path': args.path},
nb_lines=args.nb_lines,
nb_threads=args.nb_threads)
try:
ok = True
if pvn.step == 'aggregate' and not args.display:
pvn.report(args.report_style)
if pvn.step == 'init':
ok = pvn.init()
else:
ok = pvn.process()
if not ok:
raise PyvenException('Pyven build failed')
except PyvenException as e:
for msg in e.args:
Logger.get().error(msg)
sys.exit(1)
finally:
if pvn.step not in ['aggregate']:
pvn.report(args.report_style)
if args.display:
pvn.display()
toc = time.time()
Logger.get().info('Total process time : ' + str(round(toc - tic, 3)) +
' seconds')