def test_calculate_source_version_caches_computed_version(self):
self._mock_git_commands_output()
first_return_val = autoversioning.get_version()
num_check_output_calls = self.check_output_mock.call_count
second_return_val = autoversioning.get_version()
self.assertEqual(first_return_val, second_return_val,
'get_version() should return the same version across multiple calls.')
self.assertEqual(num_check_output_calls, self.check_output_mock.call_count,
'No calls to check_output() should occur after the first get_version() call.')
def test_package_version_file_absent_and_unexpected_failure_in_git_command_sets_patch_version_to_default(self):
self.check_output_mock.side_effect = [subprocess.CalledProcessError(1, 'fake')] # make all git commands fail
actual_version = autoversioning.get_version()
self.assertEqual(actual_version, '0.0.0', 'get_version() should not raise exception if git commands fail, '
'and should just set the patch version to default "0.0.0".')
def test_unexpected_failure_in_git_command_sets_patch_version_to_unknown(self):
self.check_output_mock.side_effect = [subprocess.CalledProcessError(1, 'fake')] # make all git commands fail
actual_version = autoversioning.get_version()
self.assertEqual(actual_version, '1.0.???', 'get_version() should not raise exception if git commands fail, '
'and should just set the patch version to "???".')
def test_get_version_returns_frozen_version_when_run_from_frozen_package(self):
self.patch('app.util.autoversioning._get_frozen_package_version').return_value = '1.2.3'
self.check_output_mock.side_effect = [subprocess.CalledProcessError(1, 'fake')] # make all git commands fail
actual_version = autoversioning.get_version()
self.assertEqual(actual_version, '1.2.3', 'get_version() should return get_frozen_package_version()')
def test_get_version_returns_frozen_version_when_run_from_frozen_package(self):
self.patch('app.util.autoversioning.sys').frozen = True
package_version.version = '1.2.3' # package_version is written during freeze, so this is the "frozen" version.
actual_version = autoversioning.get_version()
self.assertFalse(self.check_output_mock.called, 'No subprocess call should be necessary to get frozen version.')
self.assertEqual(actual_version, '1.2.3', 'get_version() should return what is written in package_version.py.')
def test_get_version_returns_frozen_version_when_run_from_frozen_package(self):
# package_version is written during freeze, so this is the "frozen" version.
self.patch('app.util.autoversioning._get_frozen_package_version').return_value = '1.2.3'
actual_version = autoversioning.get_version()
self.assertFalse(self.check_output_mock.called, 'No subprocess call should be necessary to get frozen version.')
self.assertEqual(actual_version, '1.2.3', 'get_version() should return what is written in package_version.py.')
def test_calculated_source_version_is_correct_when(self,
commit_is_on_trunk,
has_changed_files,
expected_version):
self._mock_git_commands_output(commit_is_on_trunk, has_changed_files)
actual_version = autoversioning.get_version()
self.assertEqual(actual_version, expected_version)
def test_unexpected_failure_in_git_command_sets_patch_version_to_unknown(
self):
self.check_output_mock.side_effect = [
subprocess.CalledProcessError(1, 'fake')
] # make all git commands fail
actual_version = autoversioning.get_version()
self.assertEqual(
actual_version, '1.0.???',
'get_version() should not raise exception if git commands fail, '
'and should just set the patch version to "???".')
def test_get_version_returns_frozen_version_when_run_from_frozen_package(
self):
self.patch('app.util.autoversioning._get_frozen_package_version'
).return_value = '1.2.3'
self.check_output_mock.side_effect = [
subprocess.CalledProcessError(1, 'fake')
] # make all git commands fail
actual_version = autoversioning.get_version()
self.assertEqual(
actual_version, '1.2.3',
'get_version() should return get_frozen_package_version()')
def application_summary(logfile_count):
"""
Return a string summarizing general info about the application. This will be output at the start of every logfile.
:param logfile_count: The number of logfiles the application has created during its current execution
:type logfile_count: int
"""
separator = '*' * 50
summary_lines = [
' ClusterRunner',
' * Version: {}'.format(autoversioning.get_version()),
]
if logfile_count > 1:
summary_lines.append(' * Logfile count: {}'.format(logfile_count))
return '\n{0}\n{1}\n{0}\n'.format(separator, '\n'.join(summary_lines))
def application_summary(logfile_count):
"""
Return a string summarizing general info about the application. This will be output at the start of every logfile.
:param logfile_count: The number of logfiles the application has created during its current execution
:type logfile_count: int
"""
separator = '*' * 50
summary_lines = [
' ClusterRunner',
' * Version: {}'.format(autoversioning.get_version()),
' * PID: {}'.format(os.getpid()),
]
if logfile_count > 1:
summary_lines.append(' * Logfile count: {}'.format(logfile_count))
return '\n{0}\n{1}\n{0}\n'.format(separator, '\n'.join(summary_lines))
def _parse_args(args):
parser = ClusterRunnerArgumentParser()
parser.add_argument('-V',
'--version',
action='version',
version='ClusterRunner ' +
autoversioning.get_version())
subparsers = parser.add_subparsers(
title='Commands',
description=
'See "{} <command> --help" for more info on a specific command.'.
format(sys.argv[0]),
dest='subcommand',
)
subparsers.required = True
# arguments specific to master
master_parser = subparsers.add_parser(
'master',
help='Run a ClusterRunner master service.',
formatter_class=ClusterRunnerHelpFormatter)
master_parser.add_argument(
'-p',
'--port',
type=int,
help='the port on which to run the master service. '
'This will be read from conf if unspecified, and defaults to 43000')
master_parser.set_defaults(subcommand_class=MasterSubcommand)
# arguments specific to slave
slave_parser = subparsers.add_parser(
'slave',
help='Run a ClusterRunner slave service.',
formatter_class=ClusterRunnerHelpFormatter)
slave_parser.add_argument(
'-p',
'--port',
type=int,
help='the port on which to run the slave service. '
'This will be read from conf if unspecified, and defaults to 43001')
slave_parser.add_argument(
'-m',
'--master-url',
help=
'the url of the master service with which the slave should communicate'
)
slave_parser.add_argument(
'-e',
'--num-executors',
type=int,
help='the number of executors to use, defaults to 30')
slave_parser.set_defaults(subcommand_class=SlaveSubcommand)
# arguments specific to both master and slave
for subparser in (master_parser, slave_parser):
subparser.add_argument(
'--eventlog-file',
help=
'change the file that eventlogs are written to, or "STDOUT" to log to stdout'
)
# arguments specific to the 'stop' subcommand
stop_parser = subparsers.add_parser(
'stop',
help='Stop all ClusterRunner services running on this host.',
formatter_class=ClusterRunnerHelpFormatter)
stop_parser.set_defaults(subcommand_class=StopSubcommand)
# arguments specific to the 'deploy' subcommand
deploy_parser = subparsers.add_parser(
'deploy',
help='Deploy clusterrunner to master and slaves.',
formatter_class=ClusterRunnerHelpFormatter)
deploy_parser.add_argument(
'-m',
'--master',
type=str,
help=
'The master host url (no port) to deploy the master on. This will be read from conf if unspecified, '
+ 'and defaults to localhost.')
deploy_parser.add_argument(
'--master-port',
type=int,
help='The port on which the master service will run. ' +
'This will be read from conf if unspecified, and defaults to 43000.')
deploy_parser.add_argument(
'-s',
'--slaves',
type=str,
nargs='+',
help=
'The space separated list of host urls (without ports) to be deployed as slaves.'
)
deploy_parser.add_argument(
'--slave-port',
type=int,
help='The port on which all of the slave services will run. ' +
'This will be read from conf if unspecified, and defaults to 43001.')
deploy_parser.add_argument(
'-n',
'--num-executors',
type=int,
help='The number of executors to use per slave, defaults to 30.')
deploy_parser.set_defaults(subcommand_class=DeploySubcommand)
# arguments specific to execute-build mode
build_parser = subparsers.add_parser(
'build',
help='Execute a build and wait for it to complete.',
formatter_class=ClusterRunnerHelpFormatter)
build_parser.add_argument(
'--master-url',
help='the url of the ClusterRunner master that will execute this build.'
)
build_parser.add_argument('-j',
'--job-name',
help='the name of the job to run')
build_parser.add_argument(
'-f',
'--remote-file',
default=None,
help=
'remote file to use in the project with the format of: <NAME> <URL>',
action='append',
nargs=2)
_add_project_type_subparsers(build_parser)
build_parser.set_defaults(subcommand_class=BuildSubcommand)
shutdown_parser = subparsers.add_parser(
'shutdown',
help=
'Put slaves in shutdown mode so they can be terminated safely. Slaves in shutdown mode will finish any '
+ 'subjobs they are currently executing, then die.',
formatter_class=ClusterRunnerHelpFormatter)
shutdown_parser.add_argument(
'-m', '--master-url', help='The url of the master, including the port')
shutdown_parser.add_argument('-a',
'--all-slaves',
action='store_true',
help='Shutdown all slaves')
shutdown_parser.add_argument('-s',
'--slave-id',
action='append',
dest='slave_ids',
help='A slave id to shut down.')
shutdown_parser.set_defaults(subcommand_class=ShutdownSubcommand)
for subparser in (master_parser, slave_parser, build_parser, stop_parser,
deploy_parser, shutdown_parser):
subparser.add_argument('-v',
'--verbose',
action='store_const',
const='DEBUG',
dest='log_level',
help='set the log level to "debug"')
subparser.add_argument('-q',
'--quiet',
action='store_const',
const='ERROR',
dest='log_level',
help='set the log level to "error"')
subparser.add_argument(
'-c',
'--config-file',
help=
'The location of the clusterrunner config file, defaults to ~/.clusterrunner/clusterrunner.conf'
)
parsed_args = vars(
parser.parse_args(args)) # vars() converts the namespace to a dict
return parsed_args
def configure_defaults(self, conf):
"""
This is the base configuration. All default configuration values belong here. These values may be overridden by
other configurations.
:type conf: Configuration
"""
if getattr(sys, 'frozen', False):
root_directory = dirname(sys.executable) # frozen
else:
root_directory = dirname(dirname(dirname(dirname(realpath(__file__))))) # unfrozen
conf.set('secret', None) # This must be overridden by conf or the service will not start for security reasons
conf.set('root_directory', root_directory) # the root directory of the application
conf.set('main_executable_path', sys.argv[0])
conf.set('version', autoversioning.get_version())
# If the user installed ClusterRunner manually, the directories used by the manual install should take
# precedence over the default install location (the home directory).
static_configured_base_directory = join('/var', 'lib', 'clusterrunner')
if isdir(static_configured_base_directory):
base_directory = static_configured_base_directory
else:
base_directory = join(expanduser('~/'), '.clusterrunner')
# where all of the clusterrunner specific files will be stored (other than source code)
conf.set('base_directory', base_directory)
# the path to the clusterrunner config file. We have to specify this in defaults since it cannot depend on
# values in the file it refers to.
conf.set('config_file', join(base_directory, 'clusterrunner.conf'))
# Where the clusterrunner service will save the process id to. These settings are set in base_config_loader
# and not in master_config_loader and slave_config_loader because CLI tools such as "clusterrunner stop"
# needs to read in these settings.
conf.set('master_pid_file', join(base_directory, '.clusterrunner_master.pid'))
conf.set('slave_pid_file', join(base_directory, '.clusterrunner_slave.pid'))
# contains symlinks to build-specific repos
conf.set('build_symlink_directory', join('/tmp', 'clusterrunner_build_symlinks'))
# where the repos are cloned to
conf.set('repo_directory', None)
conf.set('project_yaml_filename', 'clusterrunner.yaml')
conf.set('log_file', None)
conf.set('log_level', 'DEBUG')
conf.set('max_log_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_log_file_backups', 5)
# set eventlog file conf values to None to disable eventlogs by default
conf.set('log_filename', 'clusterrunner_default.log')
conf.set('eventlog_filename', 'eventlog_default.log')
conf.set('eventlog_file', None)
conf.set('max_eventlog_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_eventlog_file_backups', 5)
conf.set('hostname', platform.node())
conf.set('master_hostname', 'localhost')
conf.set('master_port', '43000')
conf.set('slaves', ['localhost'])
# Strict host key checking on git remote operations, disabled by default
conf.set('git_strict_host_key_checking', False)
# CORS support - a regex to match against allowed API request origins, or None to disable CORS
conf.set('cors_allowed_origins_regex', None)
# Helper executables
bin_dir = join(root_directory, 'bin')
conf.set('git_askpass_exe', join(bin_dir, 'git_askpass.sh'))
conf.set('git_ssh_exe', join(bin_dir, 'git_ssh.sh'))
# How slaves get the project
# Slaves would get the project from master if set to True. Otherwise it would just get the project in
# the same way how the master gets the project.
conf.set('get_project_from_master', True)
base = 'Console'
executable_name = 'clusterrunner.exe' if is_windows() else 'clusterrunner'
executables = [
Executable('main.py', base=base, targetName=executable_name)
]
if sys.platform.startswith('linux'):
# Fixes compatibility between rhel and ubuntu
bin_includes = ['/usr/lib64/libssl.so.10', '/usr/lib64/libcrypto.so.10']
file_exists = [os.path.isfile(filename) for filename in bin_includes]
if all(file_exists):
buildOptions['bin_includes'] = bin_includes
version = autoversioning.get_version()
autoversioning.write_package_version_file(version)
setup(name='ClusterRunner',
version=version,
description='',
options=dict(build_exe=buildOptions),
executables=executables)
autoversioning.restore_original_package_version_file()
if sys.platform == 'darwin':
# Fix a cx_freeze issue on mac.
# (See similar fix at https://bitbucket.org/iep-project/iep/commits/1e845c0f35)
abs_python_path = None
clusterrunner_path = join(dirname(__file__), 'dist', executable_name)
def _parse_args(args):
parser = ClusterRunnerArgumentParser()
parser.add_argument(
'-V', '--version',
action='version', version='ClusterRunner ' + autoversioning.get_version())
subparsers = parser.add_subparsers(
title='Commands',
description='See "{} <command> --help" for more info on a specific command.'.format(sys.argv[0]),
dest='subcommand',
)
subparsers.required = True
# arguments specific to master
master_parser = subparsers.add_parser(
'master',
help='Run a ClusterRunner master service.', formatter_class=ClusterRunnerHelpFormatter)
master_parser.add_argument(
'-p', '--port',
type=int,
help='the port on which to run the master service. '
'This will be read from conf if unspecified, and defaults to 43000')
master_parser.set_defaults(subcommand_class=MasterSubcommand)
# arguments specific to slave
slave_parser = subparsers.add_parser(
'slave',
help='Run a ClusterRunner slave service.', formatter_class=ClusterRunnerHelpFormatter)
slave_parser.add_argument(
'-p', '--port',
type=int,
help='the port on which to run the slave service. '
'This will be read from conf if unspecified, and defaults to 43001')
slave_parser.add_argument(
'-m', '--master-url',
help='the url of the master service with which the slave should communicate')
slave_parser.add_argument(
'-e', '--num-executors',
type=int, help='the number of executors to use, defaults to 1')
slave_parser.set_defaults(subcommand_class=SlaveSubcommand)
# arguments specific to both master and slave
for subparser in (master_parser, slave_parser):
subparser.add_argument(
'--eventlog-file',
help='change the file that eventlogs are written to, or "STDOUT" to log to stdout')
# arguments specific to the 'stop' subcommand
stop_parser = subparsers.add_parser(
'stop',
help='Stop all ClusterRunner services running on this host.', formatter_class=ClusterRunnerHelpFormatter)
stop_parser.set_defaults(subcommand_class=StopSubcommand)
# arguments specific to the 'deploy' subcommand
deploy_parser = subparsers.add_parser(
'deploy', help='Deploy clusterrunner to master and slaves.', formatter_class=ClusterRunnerHelpFormatter)
deploy_parser.add_argument(
'-m', '--master', type=str,
help='The master host url (no port) to deploy the master on. This will be read from conf if unspecified, ' +
'and defaults to localhost.')
deploy_parser.add_argument(
'--master-port', type=int, help='The port on which the master service will run. ' +
'This will be read from conf if unspecified, and defaults to 43000.')
deploy_parser.add_argument(
'-s', '--slaves', type=str, nargs='+',
help='The space separated list of host urls (without ports) to be deployed as slaves.')
deploy_parser.add_argument(
'--slave-port', type=int, help='The port on which all of the slave services will run. ' +
'This will be read from conf if unspecified, and defaults to 43001.')
deploy_parser.add_argument(
'-n', '--num-executors', type=int, help='The number of executors to use per slave, defaults to 30.')
deploy_parser.set_defaults(subcommand_class=DeploySubcommand)
# arguments specific to execute-build mode
build_parser = subparsers.add_parser(
'build',
help='Execute a build and wait for it to complete.', formatter_class=ClusterRunnerHelpFormatter)
build_parser.add_argument(
'--master-url',
help='the url of the ClusterRunner master that will execute this build.')
build_parser.add_argument(
'-j', '--job-name',
help='the name of the job to run')
build_parser.add_argument(
'-f', '--remote-file',
default=None,
help='remote file to use in the project with the format of: <NAME> <URL>',
action='append',
nargs=2)
_add_project_type_subparsers(build_parser)
build_parser.set_defaults(subcommand_class=BuildSubcommand)
shutdown_parser = subparsers.add_parser(
'shutdown',
help='Put slaves in shutdown mode so they can be terminated safely. Slaves in shutdown mode will finish any ' +
'subjobs they are currently executing, then die.',
formatter_class=ClusterRunnerHelpFormatter
)
shutdown_parser.add_argument(
'-m', '--master-url',
help='The url of the master, including the port'
)
shutdown_parser.add_argument(
'-a', '--all-slaves',
action='store_true',
help='Shutdown all slaves'
)
shutdown_parser.add_argument(
'-s', '--slave-id',
action='append',
dest='slave_ids',
help='A slave id to shut down.'
)
shutdown_parser.set_defaults(subcommand_class=ShutdownSubcommand)
for subparser in (master_parser, slave_parser, build_parser, stop_parser, deploy_parser, shutdown_parser):
subparser.add_argument(
'-v', '--verbose',
action='store_const', const='DEBUG', dest='log_level', help='set the log level to "debug"')
subparser.add_argument(
'-q', '--quiet',
action='store_const', const='ERROR', dest='log_level', help='set the log level to "error"')
subparser.add_argument(
'-c', '--config-file',
help='The location of the clusterrunner config file, defaults to ~/.clusterrunner/clusterrunner.conf'
)
parsed_args = vars(parser.parse_args(args)) # vars() converts the namespace to a dict
return parsed_args
def test_calculated_source_version_is_correct_when(self, commit_is_on_trunk, has_changed_files, expected_version):
self._mock_git_commands_output(commit_is_on_trunk, has_changed_files)
actual_version = autoversioning.get_version()
self.assertEqual(actual_version, expected_version)
}
base = 'Console'
executable_name = 'clusterrunner.exe' if is_windows() else 'clusterrunner'
executables = [Executable('main.py', base=base, targetName=executable_name)]
if sys.platform.startswith('linux'):
# Fixes compatibility between rhel and ubuntu
bin_includes = ['/usr/lib64/libssl.so.10', '/usr/lib64/libcrypto.so.10']
file_exists = [os.path.isfile(filename) for filename in bin_includes]
if all(file_exists):
buildOptions['bin_includes'] = bin_includes
version = autoversioning.get_version()
autoversioning.write_package_version_file(version)
setup(name='ClusterRunner',
version=version,
description='',
options=dict(build_exe=buildOptions),
executables=executables)
autoversioning.restore_original_package_version_file()
if sys.platform == 'darwin':
# Fix a cx_freeze issue on mac.
# (See similar fix at https://bitbucket.org/iep-project/iep/commits/1e845c0f35)
abs_python_path = None
clusterrunner_path = join(dirname(__file__), 'dist', executable_name)
def configure_defaults(self, conf):
"""
This is the base configuration. All default configuration values belong here. These values may be overridden by
other configurations.
:type conf: Configuration
"""
if getattr(sys, 'frozen', False):
root_directory = dirname(sys.executable) # frozen
else:
root_directory = dirname(
dirname(dirname(dirname(realpath(__file__))))) # unfrozen
conf.set(
'secret', None
) # This must be overridden by conf or the service will not start for security reasons
conf.set('root_directory',
root_directory) # the root directory of the application
conf.set('main_executable_path', sys.argv[0])
conf.set('version', autoversioning.get_version())
# If the user installed ClusterRunner manually, the directories used by the manual install should take
# precedence over the default install location (the home directory).
static_configured_base_directory = join('/var', 'lib', 'clusterrunner')
if isdir(static_configured_base_directory):
base_directory = static_configured_base_directory
else:
base_directory = join(expanduser('~/'), '.clusterrunner')
# where all of the clusterrunner specific files will be stored (other than source code)
conf.set('base_directory', base_directory)
# the path to the clusterrunner config file. We have to specify this in defaults since it cannot depend on
# values in the file it refers to.
conf.set('config_file', join(base_directory, 'clusterrunner.conf'))
# Where the clusterrunner service will save the process id to. These settings are set in base_config_loader
# and not in master_config_loader and slave_config_loader because CLI tools such as "clusterrunner stop"
# needs to read in these settings.
conf.set('master_pid_file',
join(base_directory, '.clusterrunner_master.pid'))
conf.set('slave_pid_file',
join(base_directory, '.clusterrunner_slave.pid'))
# contains symlinks to build-specific repos
conf.set('build_symlink_directory',
join('/tmp', 'clusterrunner_build_symlinks'))
# where the repos are cloned to
conf.set('repo_directory', None)
conf.set('project_yaml_filename', 'clusterrunner.yaml')
conf.set('log_file', None)
conf.set('log_level', 'DEBUG')
conf.set('max_log_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_log_file_backups', 5)
# set eventlog file conf values to None to disable eventlogs by default
conf.set('log_filename', 'clusterrunner_default.log')
conf.set('eventlog_filename', 'eventlog_default.log')
conf.set('eventlog_file', None)
conf.set('max_eventlog_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_eventlog_file_backups', 5)
conf.set('hostname', platform.node())
conf.set('master_hostname', 'localhost')
conf.set('master_port', '43000')
conf.set('slaves', ['localhost'])
# Strict host key checking on git remote operations, disabled by default
conf.set('git_strict_host_key_checking', False)
# CORS support - a regex to match against allowed API request origins, or None to disable CORS
conf.set('cors_allowed_origins_regex', None)
# Helper executables
bin_dir = join(root_directory, 'bin')
conf.set('git_askpass_exe', join(bin_dir, 'git_askpass.sh'))
conf.set('git_ssh_exe', join(bin_dir, 'git_ssh.sh'))
# How slaves get the project
# Slaves would get the project from master if set to True. Otherwise it would just get the project in
# the same way how the master gets the project.
conf.set('get_project_from_master', True)
# Should we have shallow or full clones of the repository?
# The master must have full clones, as slaves fetch from the master, and one cannot fetch from a shallow clone.
conf.set('shallow_clones', False)
