class Deploy(Step):
def __init__(self, verbose, release, overwrite):
super(Deploy, self).__init__(verbose)
self.name = 'deploy'
self.checker = Checker('Deployment')
self.release = release
self.overwrite = overwrite
def process(self):
return self._process_sequential()
@Step.error_checks
def _process(self, project):
ok = True
repositories = [r for r in project.repositories.values() if (not r.release or (r.release and self.release))\
and r.name != Step.WORKSPACE.name\
and r.name != Step.LOCAL_REPO.name]
for repo in repositories:
if not repo.is_reachable():
msg = 'Repository ' + repo.name + ' --> unreachable for deployment'
Logger.get().error(msg)
self.checker.errors.append([msg])
ok = False
if ok:
for repo in repositories:
try:
for artifact in [
a for a in project.artifacts.values() if a.publish
]:
repo.publish(artifact, Step.WORKSPACE, self.overwrite)
Logger.get().info('Repository ' + repo.name +
' --> Published artifact ' +
artifact.format_name())
for package in [
p for p in project.packages.values() if p.publish
]:
repo.publish(package, Step.WORKSPACE, self.overwrite)
Logger.get().info('Repository ' + repo.name +
' --> Published package ' +
package.format_name())
except RepositoryException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
raise e
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings)
class Build(Step):
def __init__(self, verbose, warning_as_error=False, nb_threads=1):
super(Build, self).__init__(verbose)
self.name = 'build'
self.checker = Checker('Build')
self.warning_as_error = warning_as_error
self.nb_threads = nb_threads
def process(self):
return self._process_parallel()
@Step.error_checks
def _process(self, project):
Logger.get().info('Starting ' + self.name)
ok = True
for tool in project.builders:
tic = time.time()
if not tool.process(self.verbose, self.warning_as_error):
ok = False
else:
toc = time.time()
Logger.get().info('Time for ' + tool.type + ':' + tool.name +
' : ' + str(round(toc - tic, 3)) +
' seconds')
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
for project in Step.PROJECTS:
for builder in project.builders:
listings.append(builder.report_content())
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings,
enable_summary=True)
def report(self):
report = super(Build, self).report()
if report:
i = 0
nb_builders = 0
while nb_builders == 0 and i < len(Step.PROJECTS):
nb_builders += len(Step.PROJECTS[i].builders)
i += 1
report = nb_builders > 0
return report
class IntegrationTests(Step):
def __init__(self, verbose, nb_threads=1):
super(IntegrationTests, self).__init__(verbose)
self.name = 'verify'
self.checker = Checker('Integration tests')
self.nb_threads = nb_threads
self.parallelizer = Parallelizer(max_concurrent=self.nb_threads)
@Step.step
def process(self):
self.parallelizer.threads = []
for project in Step.PROJECTS:
if len(project.integration_tests) == 0:
Logger.get().warning('No integration tests configured')
else:
for test in project.integration_tests:
self.parallelizer.threads.append(
Thread(target=test.process, args=(self.verbose, )))
self.parallelizer.run()
ok = True
for project in Step.PROJECTS:
project.status = pyven.constants.STATUS[0]
for test in project.integration_tests:
if test.status in pyven.constants.STATUS[1:]:
project.status = pyven.constants.STATUS[1]
ok = False
return ok
def report_content(self):
listings = []
for project in Step.PROJECTS:
for test in project.integration_tests:
listings.append(test.report_content())
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings,
enable_summary=True)
def report(self):
report = super(IntegrationTests, self).report()
if report:
i = 0
nb_tests = 0
while nb_tests == 0 and i < len(Step.PROJECTS):
nb_tests += len(Step.PROJECTS[i].integration_tests)
i += 1
report = nb_tests > 0
return report
class Deliver(Step):
def __init__(self, verbose, location):
super(Deliver, self).__init__(verbose)
self.name = 'deliver'
self.checker = Checker('Delivery')
self.location = location
def process(self):
return self._process_sequential()
@Step.error_checks
def _process(self, project):
ok = True
Logger.get().info('Delivering to directory ' + self.location)
packages = [p for p in project.packages.values()]
for repo in [
project.repositories[p.repo] for p in packages if p.to_retrieve
]:
if not repo.is_reachable():
msg = 'Repository ' + repo.name + ' --> unreachable for delivery'
Logger.get().error(msg)
self.checker.errors.append([msg])
ok = False
if ok:
for package in packages:
if package.to_retrieve:
package.deliver(self.location,
project.repositories[package.repo])
else:
package.deliver(self.location, Step.WORKSPACE)
Logger.get().info('Delivered package : ' +
package.format_name())
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings)
class Preprocess(Step):
def __init__(self, verbose, nb_threads=1):
super(Preprocess, self).__init__(verbose)
self.name = 'preprocess'
self.checker = Checker('Preprocessing')
self.nb_threads = nb_threads
def process(self):
return self._process_parallel()
@Step.error_checks
def _process(self, project):
Logger.get().info('Starting ' + self.name)
ok = True
for tool in project.preprocessors:
tic = time.time()
if not tool.process(self.verbose):
ok = False
else:
toc = time.time()
Logger.get().info('Time for ' + tool.type + ':' + tool.name +
' : ' + str(round(toc - tic, 3)) +
' seconds')
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.status in pyven.constants.STATUS[1]:
for project in Step.PROJECTS:
for preprocessor in project.preprocessors:
listings.append(preprocessor.report_content())
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings,
enable_summary=True)
def report(self):
return self.status == pyven.constants.STATUS[1]
class PackageStep(Step):
def __init__(self, verbose, nb_threads=1):
super(PackageStep, self).__init__(verbose)
self.name = 'package'
self.checker = Checker('Packaging')
self.nb_threads = nb_threads
def process(self):
return self._process_parallel()
@Step.error_checks
def _process(self, project):
ok = retrieve('artifact', project,
project.artifacts, self.checker) and retrieve(
'package', project, project.packages, self.checker)
if ok:
for package in [
p for p in project.packages.values() if not p.to_retrieve
]:
try:
if not package.pack(Step.WORKSPACE):
ok = False
except PyvenException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
ok = False
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings)
class Install(Step):
def __init__(self, verbose):
super(Install, self).__init__(verbose)
self.name = 'install'
self.checker = Checker('Installation')
def process(self):
return self._process_sequential()
@Step.error_checks
def _process(self, project):
ok = True
for artifact in [a for a in project.artifacts.values() if a.publish]:
Step.LOCAL_REPO.publish(artifact, Step.WORKSPACE)
Logger.get().info('Repository ' + Step.LOCAL_REPO.name +
' --> Published artifact ' +
artifact.format_name())
for package in [p for p in project.packages.values() if p.publish]:
Step.LOCAL_REPO.publish(package, Step.WORKSPACE)
Logger.get().info('Repository ' + Step.LOCAL_REPO.name +
' --> Published package ' +
package.format_name())
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings)
class Clean(Step):
def __init__(self, verbose, nb_threads=1):
super(Clean, self).__init__(verbose)
self.name = 'clean'
self.checker = Checker('Cleaning')
self.nb_threads = nb_threads
def process(self):
return self._process_parallel()
@Step.error_checks
def _process(self, project):
ok = True
for tool in project.builders:
if not tool.clean(self.verbose):
ok = False
for tool in project.preprocessors:
if not tool.clean(self.verbose):
ok = False
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error(self.name + ' errors found')
else:
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
for project in Step.PROJECTS:
for builder in project.builders:
listings.append(builder.report_content())
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(), status=self.report_status(), listings=listings, enable_summary=True)
class ArtifactsChecks(Step):
def __init__(self, verbose, nb_threads=1):
super(ArtifactsChecks, self).__init__(verbose)
self.name = 'artifacts checks'
self.checker = Checker('Artifacts')
self.nb_threads = nb_threads
def process(self):
return self._process_parallel()
@Step.error_checks
def _process(self, project):
Logger.get().info('Starting ' + self.name)
ok = True
for artifact in [a for a in project.artifacts.values() if not a.to_retrieve]:
if not artifact.check(self.checker):
Logger.get().error('Artifact ' + artifact.format_name() + ' --> KO')
ok = False
else:
Logger.get().info('Artifact ' + artifact.format_name() + ' --> OK')
if not ok:
project.status = pyven.constants.STATUS[1]
Logger.get().error('Artifacts missing')
else:
for artifact in [a for a in project.artifacts.values() if not a.to_retrieve]:
Step.WORKSPACE.publish(artifact, artifact.file)
project.status = pyven.constants.STATUS[0]
Logger.get().info(self.name + ' completed')
return ok
def report_content(self):
listings = []
if self.status in pyven.constants.STATUS[1]:
listings.append(self.checker.report_content())
return StepListing(title=self.title(), status=self.report_status(), listings=listings)
def report(self):
report = super(ArtifactsChecks, self).report()
if report:
i = 0
nb_artifacts = 0
while nb_artifacts == 0 and i < len(Step.PROJECTS):
nb_artifacts += len([a for a in Step.PROJECTS[i].artifacts.values() if not a.to_retrieve])
i += 1
report = nb_artifacts > 0
return report
def __init__(self, verbose, nb_threads=1):
super(Postprocess, self).__init__(verbose)
self.name = 'postprocess'
self.checker = Checker('Postprocessing')
self.nb_threads = nb_threads
def __init__(self, verbose, nb_threads=1):
super(PackageStep, self).__init__(verbose)
self.name = 'package'
self.checker = Checker('Packaging')
self.nb_threads = nb_threads
def __init__(self, verbose, nb_threads=1):
super(IntegrationTests, self).__init__(verbose)
self.name = 'verify'
self.checker = Checker('Integration tests')
self.nb_threads = nb_threads
self.parallelizer = Parallelizer(max_concurrent=self.nb_threads)
def __init__(self, verbose, nb_threads=1):
super(UnitTests, self).__init__(verbose)
self.name = 'test'
self.checker = Checker('Unit tests')
self.nb_threads = nb_threads
self.parallelizer = Parallelizer(max_concurrent=self.nb_threads)
def __init__(self, verbose, release, overwrite):
super(Deploy, self).__init__(verbose)
self.name = 'deploy'
self.checker = Checker('Deployment')
self.release = release
self.overwrite = overwrite
class Configure(Step):
def __init__(self, verbose, pym):
super(Configure, self).__init__(verbose)
self.pym = pym
self.name = 'configure'
self.checker = Checker('Configuration')
self.parsers = []
def project_title(self):
if len(self.parsers) == 0:
return ''
return self.parsers[0].parse_project_title()
def report_content(self):
listings = []
if self.checker.enabled():
listings.append(self.checker.report_content())
return StepListing(title=self.title(),
status=self.report_status(),
listings=listings)
def report(self):
return self.status == pyven.constants.STATUS[1]
@Step.step
def process(self):
project = Project(os.getcwd())
cst_file = os.path.join(os.getcwd(), 'const.xml')
if os.path.isfile(cst_file):
cst_tree = parse_xml(cst_file)
project.constants = ConstantsParser(os.getcwd()).parse(cst_tree)
return self._process(project)
@Step.error_checks
def _process(self, project):
parser = PymParser(os.path.join(project.path, self.pym),
project.plugins)
self.parsers.append(parser)
parser.parse_pym()
parser.parse_plugins()
for k, v in parser.parse_constants().items():
project.constants[k] = v
ok = self._configure_projects(project, parser)\
and self._configure_repositories(project, parser)\
and self._configure_artifacts(project, parser)\
and self._configure_packages(project, parser)\
and self._configure_processes(project, parser)
if ok:
Step.PROJECTS.append(project)
return ok
def _configure_error_checks(function):
def __intern(self, project, parser):
ok = True
try:
function(self, project, parser)
except PyvenException as e:
self.checker.errors.append(e.args)
for msg in e.args:
Logger.get().error(msg)
ok = False
return ok
return __intern
@_configure_error_checks
def _configure_projects(self, project, parser):
directories = parser.parse_projects()
for directory in directories:
if not os.path.isdir(os.path.join(project.path, directory)):
raise PyvenException('Subproject directory does not exist : ' +
directory)
else:
full_path = os.path.join(project.path, directory)
subproject = Project(full_path,
parser.plugins_manager.plugins.copy())
subproject.constants = project.constants.copy()
if not self._process(subproject):
raise PyvenException('Subproject ' + full_path +
' --> configuration failed')
Logger.get().info('Added subproject --> ' + directory)
@_configure_error_checks
def _configure_repositories(self, project, parser):
repositories = parser.parse_repositories()
project.repositories[Step.WORKSPACE.name] = Step.WORKSPACE
project.repositories[Step.LOCAL_REPO.name] = Step.LOCAL_REPO
for repo in repositories:
if repo.name == 'workspace' or repo.name == Step.LOCAL_REPO.name:
raise PyvenException('Repository name reserved --> ' +
repo.name + ' : ' + repo.url)
else:
if repo.name in project.repositories.keys():
raise PyvenException('Repository already added --> ' +
repo.name + ' : ' + repo.url)
else:
project.repositories[repo.name] = repo
if repo.is_reachable():
if repo.release:
Logger.get().info('Release repository added --> ' +
repo.name + ' : ' + repo.url)
else:
Logger.get().info('Repository added --> ' +
repo.name + ' : ' + repo.url)
else:
Logger.get().warning('Repository not accessible --> ' +
repo.name + ' : ' + repo.url)
@_configure_error_checks
def _configure_artifacts(self, project, parser):
artifacts = parser.parse_artifacts()
for artifact in artifacts:
artifact.company = project.replace_constants(artifact.company)
artifact.name = project.replace_constants(artifact.name)
artifact.config = project.replace_constants(artifact.config)
artifact.version = project.replace_constants(artifact.version)
if not artifact.to_retrieve:
artifact.file = project.replace_constants(artifact.file)
if artifact.format_name() in project.artifacts.keys():
raise PyvenException('Artifact already added --> ' +
artifact.format_name())
elif artifact.to_retrieve and artifact.repo not in project.repositories.keys(
) and artifact.repo not in [Step.LOCAL_REPO.name, 'workspace']:
raise PyvenException('Artifact repository not declared --> ' +
artifact.format_name() + ' : repo ' +
artifact.repo)
else:
project.artifacts[artifact.format_name()] = artifact
Logger.get().info('Artifact added --> ' +
artifact.format_name())
if not artifact.publish:
Logger.get().info('Artifact ' + artifact.format_name() +
' --> publishment disabled')
@_configure_error_checks
def _configure_packages(self, project, parser):
packages = parser.parse_packages()
for package in packages:
package.company = project.replace_constants(package.company)
package.name = project.replace_constants(package.name)
package.config = project.replace_constants(package.config)
package.version = project.replace_constants(package.version)
if len(package.deliveries) > 0:
for i in range(len(package.deliveries)):
package.deliveries[i] = project.replace_constants(
package.deliveries[i])
items = []
items.extend(package.items)
package.items = []
if package.format_name() in project.packages.keys():
raise PyvenException('Package already added --> ' +
package.format_name())
elif package.to_retrieve and package.repo not in project.repositories.keys(
) and package.repo not in [Step.LOCAL_REPO.name, 'workspace']:
raise PyvenException('Package repository not declared --> ' +
package.format_name() + ' : repo ' +
package.repo)
else:
for item in items:
item = project.replace_constants(item)
if item not in project.artifacts.keys():
raise PyvenException('Package ' +
package.format_name() +
' : Artifact not declared --> ' +
item)
else:
package.items.append(project.artifacts[item])
Logger.get().info('Package ' + package.format_name() +
' : Artifact added --> ' + item)
project.packages[package.format_name()] = package
Logger.get().info('Package added --> ' + package.format_name())
if not package.publish:
Logger.get().info('Package ' + package.format_name() +
' --> publishment disabled')
for package in packages:
for extension in package.extensions:
extension = project.replace_constants(extension)
if extension not in project.packages.keys():
raise PyvenException(
'Package ' + package.format_name() +
' : Package extension not declared --> ' + extension)
elif project.packages[extension].to_retrieve:
raise PyvenException(
'Package ' + package.format_name() +
' : Cannot extend a package to be retrieved --> ' +
extension)
else:
package.items.extend(project.packages[extension].items)
Logger.get().info('Package ' + package.format_name() +
' : Package extension added --> ' +
extension)
@_configure_error_checks
def _configure_processes(self, project, parser):
preprocessors = parser.parse_processes('preprocess',
'build/tools/tool', project)
builders = parser.parse_processes('build', 'build/tools/tool', project)
postprocessors = parser.parse_processes('postprocess',
'build/tools/tool', project)
unit_tests = parser.parse_processes('test', 'tests/test', project)
integration_tests = parser.parse_processes('verify', 'tests/test',
project)
project.preprocessors = self._check_processes('preprocessor',
preprocessors, project)
project.builders = self._check_processes('builder', builders, project)
project.postprocessors = self._check_processes('postprocessor',
postprocessors, project)
project.unit_tests = self._check_processes('unit test', unit_tests,
project)
project.integration_tests = self._check_processes(
'integration test', integration_tests, project)
def _check_processes(self, type, processes, project):
for process in processes:
process.name = project.replace_constants(process.name)
Logger.get().info(type + ' added --> ' + process.type + ':' +
process.name)
return processes
def __init__(self, verbose, warning_as_error=False, nb_threads=1):
super(Build, self).__init__(verbose)
self.name = 'build'
self.checker = Checker('Build')
self.warning_as_error = warning_as_error
self.nb_threads = nb_threads
def __init__(self, verbose, nb_threads=1):
super(Clean, self).__init__(verbose)
self.name = 'clean'
self.checker = Checker('Cleaning')
self.nb_threads = nb_threads
def __init__(self, verbose, pym):
super(Configure, self).__init__(verbose)
self.pym = pym
self.name = 'configure'
self.checker = Checker('Configuration')
self.parsers = []
def __init__(self, verbose, nb_threads=1):
super(ArtifactsChecks, self).__init__(verbose)
self.name = 'artifacts checks'
self.checker = Checker('Artifacts')
self.nb_threads = nb_threads
def __init__(self, verbose, location):
super(Deliver, self).__init__(verbose)
self.name = 'deliver'
self.checker = Checker('Delivery')
self.location = location
def __init__(self, verbose):
super(Install, self).__init__(verbose)
self.name = 'install'
self.checker = Checker('Installation')
