def __init__(self): self._logger = get_logger(__name__) self._all_slaves_by_url = {} self._all_builds_by_id = OrderedDict() # This is an OrderedDict so we can more easily implement get_queue() self._builds_waiting_for_slaves = Queue() self._request_queue = Queue() self._request_handler = SerialRequestHandler() self._request_queue_worker_thread = SafeThread( target=self._build_preparation_loop, name='RequestHandlerLoop', daemon=True) self._request_queue_worker_thread.start() self._slave_allocation_worker_thread = SafeThread( target=self._slave_allocation_loop, name='SlaveAllocationLoop', daemon=True) self._slave_allocation_worker_thread.start() self._master_results_path = Configuration['results_directory'] # It's important that idle slaves are only in the queue once so we use OrderedSet self._idle_slaves = OrderedSetQueue() # Delete all old builds when master starts. Remove this if/when build numbers are unique across master # starts/stops if os.path.exists(self._master_results_path): shutil.rmtree(self._master_results_path) fs.create_dir(self._master_results_path)
def _get_event_handler(self): """ Retrieves the correct event handler. Returns a Stream Handler if the event should write to STDOUT, otherwise it will return a ready RotatingFileHandler. Both subclasses inherit from the StreamHandler base class. :return: Event handler :rtype: StreamHandler """ if self.filename.upper() == 'STDOUT': return StreamHandler(sys.stdout) else: fs.create_dir(os.path.dirname(self.filename)) previous_log_file_exists = os.path.exists(self.filename) event_handler = RotatingFileHandler( filename=self.filename, max_size=Configuration['max_eventlog_file_fize'], backup_count=Configuration['max_eventlog_file_backups']) if previous_log_file_exists: event_handler.perform_rollover( ) # force starting a new eventlog file on application startup return event_handler
def configure_logging(log_level=None, log_file=None, simplified_console_logs=False): """ This should be called once as early as possible in app startup to configure logging handlers and formatting. :param log_level: The level at which to record log messages (DEBUG|INFO|NOTICE|WARNING|ERROR|CRITICAL) :type log_level: str :param log_file: The file to write logs to, or None to disable logging to a file :type log_file: str | None :param simplified_console_logs: Whether or not to use the simplified logging format and coloring :type simplified_console_logs: bool """ # Set datetimes in log messages to be local timezone instead of UTC logbook.set_datetime_format('local') # Redirect standard lib logging to capture third-party logs in our log files (e.g., tornado, requests) logging.root.setLevel( logging.WARNING ) # don't include DEBUG/INFO/NOTICE-level logs from third parties logbook.compat.redirect_logging(set_root_logger_level=False) # Add a NullHandler to suppress all log messages lower than our desired log_level. (Otherwise they go to stderr.) NullHandler().push_application() log_level = log_level or Configuration['log_level'] format_string, log_colors = _LOG_FORMAT_STRING, _LOG_COLORS if simplified_console_logs: format_string, log_colors = _SIMPLIFIED_LOG_FORMAT_STRING, _SIMPLIFIED_LOG_COLORS # handler for stdout log_handler = _ColorizingStreamHandler( stream=sys.stdout, level=log_level, format_string=format_string, log_colors=log_colors, bubble=True, ) log_handler.push_application() # handler for log file if log_file: fs.create_dir(os.path.dirname(log_file)) previous_log_file_exists = os.path.exists(log_file) event_handler = _ColorizingRotatingFileHandler( filename=log_file, level=log_level, format_string=_LOG_FORMAT_STRING, log_colors=_LOG_COLORS, bubble=True, max_size=Configuration['max_log_file_size'], backup_count=Configuration['max_log_file_backups'], ) event_handler.push_application() if previous_log_file_exists: # Force application to create a new log file on startup. event_handler.perform_rollover(increment_logfile_counter=False) else: event_handler.log_application_summary()
def _load_section_from_config_file(self, config, config_filename, section): """ Load a config file and copy all the values in a particular section to the Configuration singleton :type config: Configuration :type config_filename: str :type section: str """ try: config_parsed = ConfigFile(config_filename).read_config_from_disk() except FileNotFoundError: sample_filename = join(config.get('root_directory'), 'conf', 'default_clusterrunner.conf') fs.create_dir(config.get('base_directory')) shutil.copy(sample_filename, config_filename) chmod(config_filename, ConfigFile.CONFIG_FILE_MODE) config_parsed = ConfigFile(config_filename).read_config_from_disk() if section not in config_parsed: raise InvalidConfigError('The config file {} does not contain a [{}] section' .format(config_filename, section)) clusterrunner_config = config_parsed[section] whitelisted_file_keys = self._get_config_file_whitelisted_keys() for key in clusterrunner_config: if key not in whitelisted_file_keys: raise InvalidConfigError('The config file contains an invalid key: {}'.format(key)) value = clusterrunner_config[key] self._cast_and_set(key, value, config)
def _execute_atom_command(self, atomic_command, atom_environment_vars, atom_artifact_dir): """ Run the main command for this atom. Output the command, console output and exit code to files in the atom artifact directory. :type atomic_command: str :type atom_environment_vars: dict[str, str] :type atom_artifact_dir: str """ fs_util.create_dir(atom_artifact_dir) start_time = time.time() output, exit_code = self._project_type.execute_command_in_project(atomic_command, atom_environment_vars) elapsed_time = time.time() - start_time console_output_path = os.path.join(atom_artifact_dir, Subjob.OUTPUT_FILE) fs_util.write_file(output, console_output_path) exit_code_output_path = os.path.join(atom_artifact_dir, Subjob.EXIT_CODE_FILE) fs_util.write_file(str(exit_code) + '\n', exit_code_output_path) command_output_path = os.path.join(atom_artifact_dir, Subjob.COMMAND_FILE) fs_util.write_file(str(atomic_command) + '\n', command_output_path) time_output_path = os.path.join(atom_artifact_dir, Subjob.TIMING_FILE) fs_util.write_file('{:.2f}\n'.format(elapsed_time), time_output_path)
def _setup_build(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # For backward compatibility: If a shallow repo exists, delete it. Shallow cloning is no longer supported, # it causes failures when fetching refs that depend on commits which are excluded from the shallow clone. existing_repo_is_shallow = os.path.isfile(os.path.join(self._repo_directory, '.git', 'shallow')) if existing_repo_is_shallow: if os.path.exists(self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist _, git_exit_code = self.execute_command_in_project('git rev-parse', cwd=self._repo_directory) repo_exists = git_exit_code == 0 if not repo_exists: # This is not a git repo yet, we have to clone the project. clone_command = 'git clone {} {}'. format(self._url, self._repo_directory) self._execute_git_remote_command(clone_command) # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. # Must specify the colon in 'branch:branch' so that the branch will be created locally. This is # important because it allows the slave hosts to do a git fetch from the master for this branch. fetch_command = 'git fetch --update-head-ok {0} {1}:{1}'.format(self._remote, self._branch) self._execute_git_remote_command(fetch_command, self._repo_directory) commit_hash = self._hash or 'FETCH_HEAD' reset_command = 'git reset --hard {}'.format(commit_hash) self._execute_in_repo_and_raise_on_failure(reset_command, 'Could not reset Git repo.') self._execute_in_repo_and_raise_on_failure('git clean -dfx', 'Could not clean Git repo.')
def _execute_atom_command(self, atomic_command, atom_environment_vars, atom_artifact_dir): """ Run the main command for this atom. Output the command, console output and exit code to files in the atom artifact directory. Return the exit code. :type atomic_command: str :type atom_environment_vars: dict[str, str] :type atom_artifact_dir: str :rtype: int """ fs_util.create_dir(atom_artifact_dir) start_time = time.time() output, exit_code = self._project_type.execute_command_in_project( atomic_command, atom_environment_vars) elapsed_time = time.time() - start_time console_output_path = os.path.join(atom_artifact_dir, Subjob.OUTPUT_FILE) fs_util.write_file(output, console_output_path) exit_code_output_path = os.path.join(atom_artifact_dir, Subjob.EXIT_CODE_FILE) fs_util.write_file(str(exit_code) + '\n', exit_code_output_path) command_output_path = os.path.join(atom_artifact_dir, Subjob.COMMAND_FILE) fs_util.write_file(str(atomic_command) + '\n', command_output_path) time_output_path = os.path.join(atom_artifact_dir, Subjob.TIMING_FILE) fs_util.write_file('{:.2f}\n'.format(elapsed_time), time_output_path) return exit_code
def _get_event_handler(self): """ Retrieves the correct event handler. Returns a Stream Handler if the event should write to STDOUT, otherwise it will return a ready RotatingFileHandler. Both subclasses inherit from the StreamHandler base class. :return: Event handler :rtype: StreamHandler """ if self.filename.upper() == "STDOUT": return StreamHandler(sys.stdout) else: fs.create_dir(os.path.dirname(self.filename)) previous_log_file_exists = os.path.exists(self.filename) event_handler = RotatingFileHandler( filename=self.filename, max_size=Configuration["max_eventlog_file_fize"], backup_count=Configuration["max_eventlog_file_backups"], ) if previous_log_file_exists: event_handler.perform_rollover() # force starting a new eventlog file on application startup return event_handler
def __init__(self): self._logger = get_logger(__name__) self._all_slaves_by_url = {} self._all_builds_by_id = OrderedDict() # This is an OrderedDict so we can more easily implement get_queue() self._builds_waiting_for_slaves = Queue() self._request_queue = Queue() self._request_handler = SerialRequestHandler() self._request_queue_worker_thread = SafeThread( target=self._build_preparation_loop, name='RequestHandlerLoop', daemon=True) self._request_queue_worker_thread.start() self._slave_allocation_worker_thread = SafeThread( target=self._slave_allocation_loop, name='SlaveAllocationLoop', daemon=True) self._slave_allocation_worker_thread.start() self._master_results_path = Configuration['results_directory'] # It's important that idle slaves are only in the queue once so we use OrderedSet self._idle_slaves = OrderedSetQueue() # Asynchronously delete (but immediately rename) all old builds when master starts. # Remove this if/when build numbers are unique across master starts/stops if os.path.exists(self._master_results_path): fs.async_delete(self._master_results_path) fs.create_dir(self._master_results_path)
def _execute_atom_command(self, atomic_command, atom_environment_vars, atom_artifact_dir): """ Run the main command for this atom. Output the command, console output and exit code to files in the atom artifact directory. Return the exit code. :type atomic_command: str :type atom_environment_vars: dict[str, str] :type atom_artifact_dir: str :rtype: int """ fs_util.create_dir(atom_artifact_dir) # This console_output_file must be opened in 'w+b' mode in order to be interchangeable with the # TemporaryFile instance that gets instantiated in self._project_type.execute_command_in_project. with open(os.path.join(atom_artifact_dir, BuildArtifact.OUTPUT_FILE), mode='w+b') as console_output_file: start_time = time.time() _, exit_code = self._project_type.execute_command_in_project(atomic_command, atom_environment_vars, output_file=console_output_file) elapsed_time = time.time() - start_time exit_code_output_path = os.path.join(atom_artifact_dir, BuildArtifact.EXIT_CODE_FILE) fs_util.write_file(str(exit_code) + '\n', exit_code_output_path) command_output_path = os.path.join(atom_artifact_dir, BuildArtifact.COMMAND_FILE) fs_util.write_file(str(atomic_command) + '\n', command_output_path) time_output_path = os.path.join(atom_artifact_dir, BuildArtifact.TIMING_FILE) fs_util.write_file('{:.2f}\n'.format(elapsed_time), time_output_path) return exit_code
def __init__(self): self._logger = get_logger(__name__) self._master_results_path = Configuration['results_directory'] self._slave_registry = SlaveRegistry.singleton() self._scheduler_pool = BuildSchedulerPool() self._build_request_handler = BuildRequestHandler(self._scheduler_pool) self._build_request_handler.start() self._slave_allocator = SlaveAllocator(self._scheduler_pool) self._slave_allocator.start() # The best practice for determining the number of threads to use is # the number of threads per core multiplied by the number of physical # cores. So for example, with 10 cores, 2 sockets and 2 per core, the # max would be 40. # # Currently we use threads for incrementing/decrementing slave executor # counts (lock acquisition) and tearing down the slave (network IO). 32 threads should be # plenty for these tasks. In the case of heavy load, the bottle neck will be the number # of executors, not the time it takes to lock/unlock the executor counts or the number of # teardown requests. Tweak the number to find the sweet spot if you feel this is the case. self._thread_pool_executor = ThreadPoolExecutor(max_workers=32) # Asynchronously delete (but immediately rename) all old builds when master starts. # Remove this if/when build numbers are unique across master starts/stops if os.path.exists(self._master_results_path): fs.async_delete(self._master_results_path) fs.create_dir(self._master_results_path) # Configure heartbeat tracking self._unresponsive_slaves_cleanup_interval = Configuration[ 'unresponsive_slaves_cleanup_interval'] self._hb_scheduler = sched.scheduler() SlavesCollector.register_slaves_metrics_collector( lambda: self._slave_registry.get_all_slaves_by_id().values())
def execute_subjob(self, build_id, subjob_id, subjob_artifact_dir, atomic_commands): """ This is the method for executing a subjob. This performs the work required by executing the specified command, then archives the results into a single file and returns the filename. :type build_id: int :type subjob_id: int :type subjob_artifact_dir: str :type atomic_commands: list[str] :rtype: str """ self._logger.info('Executing subjob (Build {}, Subjob {})...', build_id, subjob_id) # Set the current task self._current_build_id = build_id self._current_subjob_id = subjob_id # Maintain a list of atom artifact directories for compression and sending back to master atom_artifact_dirs = [] # execute every atom and keep track of time elapsed for each for atom_id, atomic_command in enumerate(atomic_commands): atom_artifact_dir = os.path.join( subjob_artifact_dir, Subjob.ATOM_DIR_FORMAT.format(subjob_id, atom_id)) # remove and recreate the atom artifact dir shutil.rmtree(atom_artifact_dir, ignore_errors=True) fs_util.create_dir(atom_artifact_dir) atom_environment_vars = { 'ARTIFACT_DIR': atom_artifact_dir, 'ATOM_ID': atom_id, 'EXECUTOR_INDEX': self.id, } atom_artifact_dirs.append(atom_artifact_dir) self._execute_atom_command(atomic_command, atom_environment_vars, atom_artifact_dir) # Generate mapping of atom directories (for archiving) to paths in the archive file targets_to_archive_paths = { atom_dir: os.path.basename(os.path.normpath(atom_dir)) for atom_dir in atom_artifact_dirs } # zip file names must be unique for a build, so we append the subjob_id to the compressed file tarfile_path = os.path.join(subjob_artifact_dir, 'results_{}.tar.gz'.format(subjob_id)) fs_util.compress_directories(targets_to_archive_paths, tarfile_path) # Reset the current task self._current_build_id = None self._current_subjob_id = None return tarfile_path
def _load_section_from_config_file(self, config, config_filename, section): """ Load a config file and copy all the values in a particular section to the Configuration singleton :type config: Configuration :type config_filename: str :type section: str """ try: config_parsed = ConfigFile(config_filename).read_config_from_disk() except FileNotFoundError: sample_filename = join(config.get('root_directory'), 'conf', 'default_clusterrunner.conf') fs.create_dir(config.get('base_directory')) shutil.copy(sample_filename, config_filename) chmod(config_filename, ConfigFile.CONFIG_FILE_MODE) config_parsed = ConfigFile(config_filename).read_config_from_disk() if section not in config_parsed: raise InvalidConfigError( 'The config file {} does not contain a [{}] section'.format( config_filename, section)) clusterrunner_config = config_parsed[section] whitelisted_file_keys = self._get_config_file_whitelisted_keys() for key in clusterrunner_config: if key not in whitelisted_file_keys: raise InvalidConfigError( 'The config file contains an invalid key: {}'.format(key)) value = clusterrunner_config[key] self._cast_and_set(key, value, config)
def _write_config_to_disk(self, config_parsed): """ Write a data structure of parsed config values to disk in an INI-style format. :type config_parsed: ConfigObj """ fs.create_dir(os.path.dirname(self._filename)) config_parsed.write() os.chmod(self._filename, self.CONFIG_FILE_MODE)
def _fetch_project(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # If shallow_clones is set to True, then we need to specify the --depth=1 argument to all git fetch # and clone invocations. git_clone_fetch_depth_arg = '' if Configuration['shallow_clones']: git_clone_fetch_depth_arg = '--depth=1' existing_repo_is_shallow = os.path.isfile(os.path.join(self._repo_directory, '.git', 'shallow')) # If we disable shallow clones, but the existing repo is shallow, we must re-clone non-shallowly. if not Configuration['shallow_clones'] and existing_repo_is_shallow and os.path.exists(self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist try: self._execute_git_command_in_repo_and_raise_on_failure('rev-parse') # rev-parse succeeds if repo exists except RuntimeError: self._logger.notice('No valid repo in "{}". Cloning fresh from "{}".', self._repo_directory, self._url) self._execute_git_command_in_repo_and_raise_on_failure( git_command='clone {} {} {}'. format(git_clone_fetch_depth_arg, self._url, self._repo_directory), error_msg='Could not clone repo.' ) # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. self._execute_git_command_in_repo_and_raise_on_failure( git_command='fetch {} --update-head-ok {} {}'.format(git_clone_fetch_depth_arg, self._remote, self._branch), error_msg='Could not fetch specified branch "{}" from remote "{}".'.format(self._branch, self._remote) ) # Validate and convert the user-specified hash/refspec to a full git hash fetch_head_hash = self._execute_git_command_in_repo_and_raise_on_failure( git_command='rev-parse FETCH_HEAD', error_msg='Could not rev-parse FETCH_HEAD of {} to a commit hash.'.format(self._branch) ).strip() # Save this hash as a local ref. Named local refs are necessary for slaves to fetch correctly from the master. # The local ref will be passed on to slaves instead of the user-specified branch. self._local_ref = 'refs/clusterrunner/{}'.format(fetch_head_hash) self._execute_git_command_in_repo_and_raise_on_failure( git_command='update-ref {} {}'.format(self._local_ref, fetch_head_hash), error_msg='Could not update local ref.' ) # The '--' argument acts as a delimiter to differentiate values that can be "tree-ish" or a "path" self._execute_git_command_in_repo_and_raise_on_failure( git_command='reset --hard {} --'.format(fetch_head_hash), error_msg='Could not reset Git repo.' ) self._execute_git_command_in_repo_and_raise_on_failure( git_command='clean -dfx', error_msg='Could not clean Git repo.' )
def configure_logging(log_level=None, log_file=None, simplified_console_logs=False): """ This should be called once as early as possible in app startup to configure logging handlers and formatting. :param log_level: The level at which to record log messages (DEBUG|INFO|NOTICE|WARNING|ERROR|CRITICAL) :type log_level: str :param log_file: The file to write logs to, or None to disable logging to a file :type log_file: str | None :param simplified_console_logs: Whether or not to use the simplified logging format and coloring :type simplified_console_logs: bool """ # Set datetimes in log messages to be local timezone instead of UTC logbook.set_datetime_format('local') # Redirect standard lib logging to capture third-party logs in our log files (e.g., tornado, requests) logging.root.setLevel(logging.WARNING) # don't include DEBUG/INFO/NOTICE-level logs from third parties logbook.compat.redirect_logging(set_root_logger_level=False) # Add a NullHandler to suppress all log messages lower than our desired log_level. (Otherwise they go to stderr.) NullHandler().push_application() log_level = log_level or Configuration['log_level'] format_string, log_colors = _LOG_FORMAT_STRING, _LOG_COLORS if simplified_console_logs: format_string, log_colors = _SIMPLIFIED_LOG_FORMAT_STRING, _SIMPLIFIED_LOG_COLORS # handler for stdout log_handler = _ColorizingStreamHandler( stream=sys.stdout, level=log_level, format_string=format_string, log_colors=log_colors, bubble=True, ) log_handler.push_application() # handler for log file if log_file: fs.create_dir(os.path.dirname(log_file)) previous_log_file_exists = os.path.exists(log_file) event_handler = _ColorizingRotatingFileHandler( filename=log_file, level=log_level, format_string=_LOG_FORMAT_STRING, log_colors=_LOG_COLORS, bubble=True, max_size=Configuration['max_log_file_size'], backup_count=Configuration['max_log_file_backups'], ) event_handler.push_application() if previous_log_file_exists: # Force application to create a new log file on startup. event_handler.perform_rollover(increment_logfile_counter=False) else: event_handler.log_application_summary()
def _fetch_project(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # For backward compatibility: If a shallow repo exists, delete it. Shallow cloning is no longer supported, # it causes failures when fetching refs that depend on commits which are excluded from the shallow clone. existing_repo_is_shallow = os.path.isfile(os.path.join(self._repo_directory, '.git', 'shallow')) if existing_repo_is_shallow: if os.path.exists(self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist try: self._execute_git_command_in_repo_and_raise_on_failure('rev-parse') # rev-parse succeeds if repo exists except RuntimeError: self._logger.notice('No valid repo in "{}". Cloning fresh from "{}".', self._repo_directory, self._url) self._execute_git_command_in_repo_and_raise_on_failure( git_command='clone {} {}'. format(self._url, self._repo_directory), error_msg='Could not clone repo.' ) # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. self._execute_git_command_in_repo_and_raise_on_failure( git_command='fetch --update-head-ok {} {}'.format(self._remote, self._branch), error_msg='Could not fetch specified branch "{}" from remote "{}".'.format(self._branch, self._remote) ) # Validate and convert the user-specified hash/refspec to a full git hash self._hash = self._execute_git_command_in_repo_and_raise_on_failure( git_command='rev-parse {}'.format(self._hash), error_msg='Could not rev-parse "{}" to a commit hash.'.format(self._hash) ).strip() # Save this hash as a local ref. Named local refs are necessary for slaves to fetch correctly from the master. # The local ref will be passed on to slaves instead of the user-specified branch. self._local_ref = 'refs/clusterrunner/' + self._hash self._execute_git_command_in_repo_and_raise_on_failure( git_command='update-ref {} {}'.format(self._local_ref, self._hash), error_msg='Could not update local ref.' ) # The '--' argument acts as a delimiter to differentiate values that can be "tree-ish" or a "path" self._execute_git_command_in_repo_and_raise_on_failure( git_command='reset --hard {} --'.format(self._hash), error_msg='Could not reset Git repo.' ) self._execute_git_command_in_repo_and_raise_on_failure( git_command='clean -dfx', error_msg='Could not clean Git repo.' )
def execute_subjob(self, build_id, subjob_id, subjob_artifact_dir, atomic_commands): """ This is the method for executing a subjob. This performs the work required by executing the specified command, then archives the results into a single file and returns the filename. :type build_id: int :type subjob_id: int :type subjob_artifact_dir: str :type atomic_commands: list[str] :rtype: str """ self._logger.info('Executing subjob (Build {}, Subjob {})...', build_id, subjob_id) # Set the current task self._current_build_id = build_id self._current_subjob_id = subjob_id # Maintain a list of atom artifact directories for compression and sending back to master atom_artifact_dirs = [] # execute every atom and keep track of time elapsed for each for atom_id, atomic_command in enumerate(atomic_commands): atom_artifact_dir = os.path.join(subjob_artifact_dir, Subjob.ATOM_DIR_FORMAT.format(subjob_id, atom_id)) # remove and recreate the atom artifact dir shutil.rmtree(atom_artifact_dir, ignore_errors=True) fs_util.create_dir(atom_artifact_dir) atom_environment_vars = { 'ARTIFACT_DIR': atom_artifact_dir, 'ATOM_ID': atom_id, 'EXECUTOR_INDEX': self.id, } atom_artifact_dirs.append(atom_artifact_dir) self._execute_atom_command(atomic_command, atom_environment_vars, atom_artifact_dir) # Generate mapping of atom directories (for archiving) to paths in the archive file targets_to_archive_paths = {atom_dir: os.path.basename(os.path.normpath(atom_dir)) for atom_dir in atom_artifact_dirs} # zip file names must be unique for a build, so we append the subjob_id to the compressed file tarfile_path = os.path.join(subjob_artifact_dir, 'results_{}.tar.gz'.format(subjob_id)) fs_util.compress_directories(targets_to_archive_paths, tarfile_path) # Reset the current task self._current_build_id = None self._current_subjob_id = None return tarfile_path
def _load_section_from_config_file(self, config, config_filename, section): """ Load a config file and copy all the values in a particular section to the Configuration singleton :type config: Configuration :type config_filename: str :type section: str """ # Only keys from this list will be loaded from a conf file. If the conf file contains other keys we will # error to alert the user. config_key_validation = [ 'secret', 'base_directory', 'log_level', 'build_symlink_directory', 'hostname', 'slaves', 'port', 'num_executors', 'master_hostname', 'master_port', 'log_filename', 'max_log_file_size', 'eventlog_filename', 'git_strict_host_key_checking', 'cors_allowed_origins_regex', ] try: config_parsed = ConfigFile(config_filename).read_config_from_disk() except FileNotFoundError: sample_filename = join(config.get('root_directory'), 'conf', 'default_clusterrunner.conf') fs.create_dir(config.get('base_directory')) shutil.copy(sample_filename, config_filename) chmod(config_filename, ConfigFile.CONFIG_FILE_MODE) config_parsed = ConfigFile(config_filename).read_config_from_disk() if section not in config_parsed: raise _InvalidConfigError( 'The config file {} does not contain a [{}] section'.format( config_filename, section)) clusterrunner_config = config_parsed[section] for key in clusterrunner_config: if key not in config_key_validation: raise _InvalidConfigError( 'The config file contains an invalid key: {}'.format(key)) value = clusterrunner_config[key] self._cast_and_set(key, value, config)
def __init__(self): self._logger = get_logger(__name__) self._master_results_path = Configuration['results_directory'] self._all_slaves_by_url = {} self._all_builds_by_id = OrderedDict() self._build_request_handler = BuildRequestHandler() self._build_request_handler.start() self._slave_allocator = SlaveAllocator(self._build_request_handler) self._slave_allocator.start() # Asynchronously delete (but immediately rename) all old builds when master starts. # Remove this if/when build numbers are unique across master starts/stops if os.path.exists(self._master_results_path): fs.async_delete(self._master_results_path) fs.create_dir(self._master_results_path)
def __init__(self): self._logger = get_logger(__name__) self._master_results_path = Configuration["results_directory"] self._all_slaves_by_url = {} self._all_builds_by_id = OrderedDict() self._build_request_handler = BuildRequestHandler() self._build_request_handler.start() self._slave_allocator = SlaveAllocator(self._build_request_handler) self._slave_allocator.start() # Asynchronously delete (but immediately rename) all old builds when master starts. # Remove this if/when build numbers are unique across master starts/stops if os.path.exists(self._master_results_path): fs.async_delete(self._master_results_path) fs.create_dir(self._master_results_path)
def _load_section_from_config_file(self, config, config_filename, section): """ Load a config file and copy all the values in a particular section to the Configuration singleton :type config: Configuration :type config_filename: str :type section: str """ # Only keys from this list will be loaded from a conf file. If the conf file contains other keys we will # error to alert the user. config_key_validation = [ 'secret', 'base_directory', 'log_level', 'build_symlink_directory', 'hostname', 'slaves', 'port', 'num_executors', 'master_hostname', 'master_port', 'log_filename', 'max_log_file_size', 'eventlog_filename', 'git_strict_host_key_checking', 'cors_allowed_origins_regex', ] try: config_parsed = ConfigFile(config_filename).read_config_from_disk() except FileNotFoundError: sample_filename = join(config.get('root_directory'), 'conf', 'default_clusterrunner.conf') fs.create_dir(config.get('base_directory')) shutil.copy(sample_filename, config_filename) chmod(config_filename, ConfigFile.CONFIG_FILE_MODE) config_parsed = ConfigFile(config_filename).read_config_from_disk() if section not in config_parsed: raise _InvalidConfigError('The config file {} does not contain a [{}] section' .format(config_filename, section)) clusterrunner_config = config_parsed[section] for key in clusterrunner_config: if key not in config_key_validation: raise _InvalidConfigError('The config file contains an invalid key: {}'.format(key)) value = clusterrunner_config[key] self._cast_and_set(key, value, config)
def initialize(eventlog_file=None): """ Initialize the analytics output. This will cause analytics events to be output to either a file or stdout. If this function is not called, analytics events will not be output. If it is called with a filename, the events will be output to that file. If it is called with 'STDOUT' or None, the events will be output to stdout. :param eventlog_file: The filename to output events to, 'STDOUT' to output to stdout, None to disable event logging :type eventlog_file: str | None """ global _analytics_logger, _eventlog_file _eventlog_file = eventlog_file if not eventlog_file: _analytics_logger = None return if eventlog_file.upper() == 'STDOUT': event_handler = StreamHandler(sys.stdout) else: fs.create_dir(os.path.dirname(eventlog_file)) previous_log_file_exists = os.path.exists(eventlog_file) event_handler = RotatingFileHandler( filename=eventlog_file, max_size=Configuration['max_eventlog_file_size'], backup_count=Configuration['max_eventlog_file_backups'], ) if previous_log_file_exists: event_handler.perform_rollover( ) # force starting a new eventlog file on application startup event_handler.format_string = '{record.message}' # only output raw log message -- no timestamp or log level handler = TaggingHandler( {'event': event_handler }, # enable logging to the event_handler with the event() method bubble=True, ) handler.push_application() _analytics_logger = TaggingLogger('analytics', ['event'])
def _fetch_project(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # For backward compatibility: If a shallow repo exists, delete it. Shallow cloning is no longer supported, # it causes failures when fetching refs that depend on commits which are excluded from the shallow clone. existing_repo_is_shallow = os.path.isfile(os.path.join(self._repo_directory, '.git', 'shallow')) if existing_repo_is_shallow: if os.path.exists(self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist _, git_exit_code = self.execute_command_in_project('git rev-parse', cwd=self._repo_directory) repo_exists = git_exit_code == 0 if not repo_exists: # This is not a git repo yet, we have to clone the project. clone_command = 'git clone {} {}'. format(self._url, self._repo_directory) self._git_remote_command_executor.execute(clone_command) # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. fetch_command = 'git fetch --update-head-ok {} {}'.format(self._remote, self._branch) self._git_remote_command_executor.execute(fetch_command, cwd=self._repo_directory) # Validate and convert the user-specified hash/refspec to a full git hash self._hash = self._execute_in_repo_and_raise_on_failure( 'git rev-parse {}'.format(self._hash), 'Could not rev-parse "{}" to a commit hash.'.format(self._hash) ).strip() # Save this hash as a local ref. Named local refs are necessary for slaves to fetch correctly from the master. # The local ref will be passed on to slaves instead of the user-specified branch. self._local_ref = 'refs/clusterrunner/' + self._hash update_ref_command = 'git update-ref {} {}'.format(self._local_ref, self._hash) self._execute_in_repo_and_raise_on_failure(update_ref_command, 'Could not update local ref.') # The '--' option acts as a delimiter to differentiate values that can be "tree-ish" or a "path" reset_command = 'git reset --hard {} --'.format(self._hash) self._execute_in_repo_and_raise_on_failure(reset_command, 'Could not reset Git repo.') self._execute_in_repo_and_raise_on_failure('git clean -dfx', 'Could not clean Git repo.')
def __init__(self, url, build_project_directory='', project_directory='', remote='origin', branch='master', hash=None, config=None, job_name=None, remote_files=None): """ Note: the first line of each parameter docstring will be exposed as command line argument documentation for the clusterrunner build client. :param url: url to the git repo (ie: https, ssh) :type url: str :param build_project_directory: the symlinked directory of where PROJECT_DIR should end up being set to :type build_project_directory: str :param project_directory: path within the repo that contains cluster_runner.yaml :type project_directory: str :param remote: The git remote name to fetch from :type remote: str :param branch: The git branch name on the remote to fetch :type branch: str :param hash: The hash to reset hard on. If both hash and branch are set, we only use the hash. :type hash: str :param config: a yaml string representing the project_type's config :type config: str|None :param job_name: a list of job names we intend to run :type job_name: list [str] | None :param remote_files: dictionary mapping of output file to URL :type remote_files: dict[str, str] | None """ super().__init__(config, job_name, remote_files) self._url = url self._remote = remote self._branch = branch self._hash = hash url_components = urlparse(url) url_full_path_parts = url_components.path.split('/') repo_name = url_full_path_parts[-1].split('.')[0] url_folder_path_parts = url_full_path_parts[:-1] repo_directory = os.path.join(Configuration['repo_directory'], url_components.netloc, *url_folder_path_parts) self._repo_directory = os.path.join(repo_directory, repo_name) self._timing_file_directory = os.path.join( Configuration['timings_directory'], url_components.netloc, url_components.path.strip('/') ) # We explicitly set the repo directory to 700 so we don't inadvertently expose the repo to access by other users fs.create_dir(self._repo_directory, 0o700) fs.create_dir(self._timing_file_directory, 0o700) fs.create_dir(os.path.dirname(build_project_directory)) # Create a symlink from the generated build project directory to the actual project directory. # This is done in order to switch between the master's and the slave's copies of the repo while not # having to do something hacky in order to user the master's generated atoms on the slaves. actual_project_directory = os.path.join(self._repo_directory, project_directory) try: os.unlink(build_project_directory) except FileNotFoundError: pass os.symlink(actual_project_directory, build_project_directory) self.project_directory = build_project_directory
def _fetch_project(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # For backward compatibility: If a shallow repo exists, delete it. Shallow cloning is no longer supported, # it causes failures when fetching refs that depend on commits which are excluded from the shallow clone. existing_repo_is_shallow = os.path.isfile( os.path.join(self._repo_directory, '.git', 'shallow')) if existing_repo_is_shallow: if os.path.exists(self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist _, git_exit_code = self.execute_command_in_project( 'git rev-parse', cwd=self._repo_directory) repo_exists = git_exit_code == 0 if not repo_exists: # This is not a git repo yet, we have to clone the project. clone_command = 'git clone {} {}'.format(self._url, self._repo_directory) self._git_remote_command_executor.execute(clone_command) # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. # Must specify the colon in 'branch:branch' so that the branch will be created locally. This is # important because it allows the slave hosts to do a git fetch from the master for this branch. fetch_command = 'git fetch --update-head-ok {0} {1}:{1}'.format( self._remote, self._branch) self._git_remote_command_executor.execute(fetch_command, cwd=self._repo_directory) commit_hash = self._hash or 'FETCH_HEAD' # The '--' option acts as a delimiter to differentiate values that can be "tree-ish" or a "path" reset_command = 'git reset --hard {} --'.format(commit_hash) self._execute_in_repo_and_raise_on_failure( reset_command, 'Could not reset Git repo.') self._execute_in_repo_and_raise_on_failure( 'git clean -dfx', 'Could not clean Git repo.')
def __init__(self, url, build_project_directory='', project_directory='', remote='origin', branch='master', hash=None, config=None, job_name=None, remote_files=None): """ Note: the first line of each parameter docstring will be exposed as command line argument documentation for the clusterrunner build client. :param url: url to the git repo (ie: https, ssh) :type url: str :param build_project_directory: the symlinked directory of where PROJECT_DIR should end up being set to :type build_project_directory: str :param project_directory: path within the repo that contains cluster_runner.yaml :type project_directory: str :param remote: The git remote name to fetch from :type remote: str :param branch: The git branch name on the remote to fetch :type branch: str :param hash: The hash to reset hard on. If both hash and branch are set, we only use the hash. :type hash: str :param config: a yaml string representing the project_type's config :type config: str|None :param job_name: a list of job names we intend to run :type job_name: list [str] | None :param remote_files: dictionary mapping of output file to URL :type remote_files: dict[str, str] | None """ super().__init__(config, job_name, remote_files) self._url = url self._remote = remote self._branch = branch self._hash = hash self._repo_directory = self.get_full_repo_directory(self._url) self._timing_file_directory = self.get_timing_file_directory(self._url) self._git_remote_command_executor = _GitRemoteCommandExecutor() # We explicitly set the repo directory to 700 so we don't inadvertently expose the repo to access by other users fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) fs.create_dir(self._timing_file_directory, self.DIRECTORY_PERMISSIONS) fs.create_dir(os.path.dirname(build_project_directory)) # Create a symlink from the generated build project directory to the actual project directory. # This is done in order to switch between the master's and the slave's copies of the repo while not # having to do something hacky in order to user the master's generated atoms on the slaves. actual_project_directory = os.path.join(self._repo_directory, project_directory) try: os.unlink(build_project_directory) except FileNotFoundError: pass os.symlink(actual_project_directory, build_project_directory) self.project_directory = build_project_directory
def initialize(eventlog_file=None): """ Initialize the analytics output. This will cause analytics events to be output to either a file or stdout. If this function is not called, analytics events will not be output. If it is called with a filename, the events will be output to that file. If it is called with 'STDOUT' or None, the events will be output to stdout. :param eventlog_file: The filename to output events to, 'STDOUT' to output to stdout, None to disable event logging :type eventlog_file: str | None """ global _analytics_logger, _eventlog_file _eventlog_file = eventlog_file if not eventlog_file: _analytics_logger = None return if eventlog_file.upper() == 'STDOUT': event_handler = StreamHandler(sys.stdout) else: fs.create_dir(os.path.dirname(eventlog_file)) previous_log_file_exists = os.path.exists(eventlog_file) event_handler = RotatingFileHandler( filename=eventlog_file, max_size=Configuration['max_eventlog_file_size'], backup_count=Configuration['max_eventlog_file_backups'], ) if previous_log_file_exists: event_handler.perform_rollover() # force starting a new eventlog file on application startup event_handler.format_string = '{record.message}' # only output raw log message -- no timestamp or log level handler = TaggingHandler( {'event': event_handler}, # enable logging to the event_handler with the event() method bubble=True, ) handler.push_application() _analytics_logger = TaggingLogger('analytics', ['event'])
def _execute_atom_command(self, atomic_command, atom_environment_vars, atom_artifact_dir): """ Run the main command for this atom. Output the command, console output and exit code to files in the atom artifact directory. Return the exit code. :type atomic_command: str :type atom_environment_vars: dict[str, str] :type atom_artifact_dir: str :rtype: int """ fs_util.create_dir(atom_artifact_dir) # This console_output_file must be opened in 'w+b' mode in order to be interchangeable with the # TemporaryFile instance that gets instantiated in self._project_type.execute_command_in_project. with open(os.path.join(atom_artifact_dir, BuildArtifact.OUTPUT_FILE), mode='w+b') as console_output_file: start_time = time.time() _, exit_code = self._project_type.execute_command_in_project( atomic_command, atom_environment_vars, output_file=console_output_file) elapsed_time = time.time() - start_time exit_code_output_path = os.path.join(atom_artifact_dir, BuildArtifact.EXIT_CODE_FILE) fs_util.write_file(str(exit_code) + '\n', exit_code_output_path) command_output_path = os.path.join(atom_artifact_dir, BuildArtifact.COMMAND_FILE) fs_util.write_file(str(atomic_command) + '\n', command_output_path) time_output_path = os.path.join(atom_artifact_dir, BuildArtifact.TIMING_FILE) fs_util.write_file('{:.2f}\n'.format(elapsed_time), time_output_path) return exit_code
def __init__(self, url, build_project_directory='', project_directory='', remote='origin', branch='master', hash='FETCH_HEAD', config=None, job_name=None, remote_files=None, atoms_override=None): """ Note: the first line of each parameter docstring will be exposed as command line argument documentation for the clusterrunner build client. :param url: url to the git repo (ie: https, ssh) :type url: str :param build_project_directory: the symlinked directory of where PROJECT_DIR should end up being set to :type build_project_directory: str :param project_directory: path within the repo that contains clusterrunner.yaml :type project_directory: str :param remote: The git remote name to fetch from :type remote: str :param branch: The git branch name on the remote to fetch :type branch: str :param hash: The hash to reset hard on. If hash is not set, we use the FETCH_HEAD of <branch>. :type hash: str :param config: a yaml string representing the project_type's config :type config: str|None :param job_name: a list of job names we intend to run :type job_name: list [str] | None :param remote_files: dictionary mapping of output file to URL :type remote_files: dict[str, str] | None :param atoms_override: The list of overridden atoms (if specified, will not run atomizer). :type atoms_override: list[str] | None """ super().__init__(config, job_name, remote_files, atoms_override) self._url = url self._remote = remote self._branch = branch self._hash = hash self._repo_directory = self.get_full_repo_directory(self._url) self._timing_file_directory = self.get_timing_file_directory(self._url) self._local_ref = None self._logger = log.get_logger(__name__) # We explicitly set the repo directory to 700 so we don't inadvertently expose the repo to access by other users fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) fs.create_dir(self._timing_file_directory, self.DIRECTORY_PERMISSIONS) fs.create_dir(os.path.dirname(build_project_directory)) # Create a symlink from the generated build project directory to the actual project directory. # This is done in order to switch between the master's and the slave's copies of the repo while not # having to do something hacky in order to user the master's generated atoms on the slaves. actual_project_directory = os.path.join(self._repo_directory, project_directory) try: os.unlink(build_project_directory) except FileNotFoundError: pass os.symlink(actual_project_directory, build_project_directory) self.project_directory = build_project_directory
def execute_subjob(self, build_id, subjob_id, atomic_commands, base_executor_index): """ This is the method for executing a subjob. This performs the work required by executing the specified command, then archives the results into a single file and returns the filename. :type build_id: int :type subjob_id: int :type atomic_commands: list[str] :type base_executor_index: int :rtype: str """ self._logger.info('Executing subjob (Build {}, Subjob {})...', build_id, subjob_id) # Set the current task self._current_build_id = build_id self._current_subjob_id = subjob_id # Maintain a list of atom artifact directories for compression and sending back to master atom_artifact_dirs = [] # execute every atom and keep track of time elapsed for each for atom_id, atomic_command in enumerate(atomic_commands): atom_artifact_dir = BuildArtifact.atom_artifact_directory( build_id, subjob_id, atom_id, result_root=Configuration['artifact_directory']) # remove and recreate the atom artifact dir shutil.rmtree(atom_artifact_dir, ignore_errors=True) fs_util.create_dir(atom_artifact_dir) atom_environment_vars = { 'ARTIFACT_DIR': atom_artifact_dir, 'ATOM_ID': atom_id, 'EXECUTOR_INDEX': self.id, # Deprecated, use MACHINE_EXECUTOR_INDEX 'MACHINE_EXECUTOR_INDEX': self.id, 'BUILD_EXECUTOR_INDEX': base_executor_index + self.id, } atom_artifact_dirs.append(atom_artifact_dir) job_name = self._project_type.job_name atom_event_data = { 'build_id': build_id, 'atom_id': atom_id, 'job_name': job_name, 'subjob_id': subjob_id } analytics.record_event(analytics.ATOM_START, **atom_event_data) exit_code = self._execute_atom_command(atomic_command, atom_environment_vars, atom_artifact_dir) atom_event_data['exit_code'] = exit_code analytics.record_event(analytics.ATOM_FINISH, **atom_event_data) # Generate mapping of atom directories (for archiving) to paths in the archive file targets_to_archive_paths = { atom_dir: os.path.basename(os.path.normpath(atom_dir)) for atom_dir in atom_artifact_dirs } # zip file names must be unique for a build, so we append the subjob_id to the compressed file subjob_artifact_dir = BuildArtifact.build_artifact_directory( build_id, result_root=Configuration['artifact_directory']) tarfile_path = os.path.join(subjob_artifact_dir, 'results_{}.tar.gz'.format(subjob_id)) fs_util.tar_directories(targets_to_archive_paths, tarfile_path) # Reset the current task self._current_build_id = None self._current_subjob_id = None return tarfile_path
def _fetch_project(self): """ Clones the project if necessary, fetches from the remote repo and resets to the requested commit """ # If shallow_clones is set to True, then we need to specify the --depth=1 argument to all git fetch # and clone invocations. git_clone_fetch_depth_arg = '' if Configuration['shallow_clones']: git_clone_fetch_depth_arg = '--depth=1' existing_repo_is_shallow = os.path.isfile( os.path.join(self._repo_directory, '.git', 'shallow')) # If we disable shallow clones, but the existing repo is shallow, we must re-clone non-shallowly. if not Configuration[ 'shallow_clones'] and existing_repo_is_shallow and os.path.exists( self._repo_directory): shutil.rmtree(self._repo_directory) fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS) # Clone the repo if it doesn't exist try: self._execute_git_command_in_repo_and_raise_on_failure( 'rev-parse') # rev-parse succeeds if repo exists except RuntimeError: self._logger.notice( 'No valid repo in "{}". Cloning fresh from "{}".', self._repo_directory, self._url) self._execute_git_command_in_repo_and_raise_on_failure( git_command='clone {} {} {}'.format(git_clone_fetch_depth_arg, self._url, self._repo_directory), error_msg='Could not clone repo.') # Must add the --update-head-ok in the scenario that the current branch of the working directory # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code. self._execute_git_command_in_repo_and_raise_on_failure( git_command='fetch {} --update-head-ok {} {}'.format( git_clone_fetch_depth_arg, self._remote, self._branch), error_msg='Could not fetch specified branch "{}" from remote "{}".' .format(self._branch, self._remote)) # Validate and convert the user-specified hash/refspec to a full git hash fetch_head_hash = self._execute_git_command_in_repo_and_raise_on_failure( git_command='rev-parse FETCH_HEAD', error_msg='Could not rev-parse FETCH_HEAD of {} to a commit hash.'. format(self._branch)).strip() # Save this hash as a local ref. Named local refs are necessary for slaves to fetch correctly from the master. # The local ref will be passed on to slaves instead of the user-specified branch. self._local_ref = 'refs/clusterrunner/{}'.format(fetch_head_hash) self._execute_git_command_in_repo_and_raise_on_failure( git_command='update-ref {} {}'.format(self._local_ref, fetch_head_hash), error_msg='Could not update local ref.') # The '--' argument acts as a delimiter to differentiate values that can be "tree-ish" or a "path" self._execute_git_command_in_repo_and_raise_on_failure( git_command='reset --hard {} --'.format(fetch_head_hash), error_msg='Could not reset Git repo.') self._execute_git_command_in_repo_and_raise_on_failure( git_command='clean -dfx', error_msg='Could not clean Git repo.')
def execute_subjob(self, build_id, subjob_id, atomic_commands, base_executor_index): """ This is the method for executing a subjob. This performs the work required by executing the specified command, then archives the results into a single file and returns the filename. :type build_id: int :type subjob_id: int :type atomic_commands: list[str] :type base_executor_index: int :rtype: str """ self._logger.info('Executing subjob (Build {}, Subjob {})...', build_id, subjob_id) # Set the current task self._current_build_id = build_id self._current_subjob_id = subjob_id # Maintain a list of atom artifact directories for compression and sending back to master atom_artifact_dirs = [] # execute every atom and keep track of time elapsed for each for atom_id, atomic_command in enumerate(atomic_commands): atom_artifact_dir = BuildArtifact.atom_artifact_directory( build_id, subjob_id, atom_id, result_root=Configuration['artifact_directory'] ) # remove and recreate the atom artifact dir shutil.rmtree(atom_artifact_dir, ignore_errors=True) fs_util.create_dir(atom_artifact_dir) atom_environment_vars = { 'ARTIFACT_DIR': atom_artifact_dir, 'ATOM_ID': atom_id, 'EXECUTOR_INDEX': self.id, # Deprecated, use MACHINE_EXECUTOR_INDEX 'MACHINE_EXECUTOR_INDEX': self.id, 'BUILD_EXECUTOR_INDEX': base_executor_index + self.id, } atom_artifact_dirs.append(atom_artifact_dir) job_name = self._project_type.job_name atom_event_data = {'build_id': build_id, 'atom_id': atom_id, 'job_name': job_name, 'subjob_id': subjob_id} analytics.record_event(analytics.ATOM_START, **atom_event_data) exit_code = self._execute_atom_command(atomic_command, atom_environment_vars, atom_artifact_dir) atom_event_data['exit_code'] = exit_code analytics.record_event(analytics.ATOM_FINISH, **atom_event_data) # Generate mapping of atom directories (for archiving) to paths in the archive file targets_to_archive_paths = {atom_dir: os.path.basename(os.path.normpath(atom_dir)) for atom_dir in atom_artifact_dirs} # zip file names must be unique for a build, so we append the subjob_id to the compressed file subjob_artifact_dir = BuildArtifact.build_artifact_directory(build_id, result_root=Configuration['artifact_directory']) tarfile_path = os.path.join(subjob_artifact_dir, 'results_{}.tar.gz'.format(subjob_id)) fs_util.compress_directories(targets_to_archive_paths, tarfile_path) # Reset the current task self._current_build_id = None self._current_subjob_id = None return tarfile_path