def execute_next_subjob_or_free_executor(self, slave):
"""
Grabs an unstarted subjob off the queue and sends it to the specified slave to be executed. If the unstarted
subjob queue is empty, we teardown the slave to free it up for other builds.
:type slave: Slave
"""
if self._build.is_canceled:
self._free_slave_executor(slave)
return
# This lock prevents the scenario where a subjob is pulled from the queue but cannot be assigned to this
# slave because it is shutdown, so we put it back on the queue but in the meantime another slave enters
# this method, finds the subjob queue empty, and is torn down. If that was the last 'living' slave, the
# build would be stuck.
with self._subjob_assignment_lock:
try:
subjob = self._build._unstarted_subjobs.get(block=False)
except Empty:
self._free_slave_executor(slave)
return
self._logger.debug('Sending {} to {}.', subjob, slave)
try:
slave.start_subjob(subjob)
subjob.mark_in_progress(slave)
except SlaveError as ex:
internal_errors.labels(ErrorType.SubjobWriteFailure).inc() # pylint: disable=no-member
self._logger.warning('Failed to start {} on {}: {}. Requeuing subjob and freeing slave executor...',
subjob, slave, repr(ex))
# An executor is currently allocated for this subjob in begin_subjob_executions_on_slave.
# Since the slave has been marked for shutdown, we need to free the executor.
self._build._unstarted_subjobs.put(subjob)
self._free_slave_executor(slave)
def atomize_in_project(self, project_type):
"""
Translate the atomizer dicts that this instance was initialized with into a list of actual atom commands. This
executes atomizer commands inside the given project in order to generate the atoms.
:param project_type: The ProjectType instance in which to execute the atomizer commands
:type project_type: ProjectType
:return: The list of environment variable "export" atom commands
:rtype: list[app.master.atom.Atom]
"""
atoms_list = []
for atomizer_dict in self._atomizer_dicts:
for atomizer_var_name, atomizer_command in atomizer_dict.items():
atomizer_output, exit_code = project_type.execute_command_in_project(atomizer_command)
if exit_code != 0:
self._logger.error('Atomizer command "{}" for variable "{}" failed with exit code: {} and output:'
'\n{}', atomizer_command, atomizer_var_name, exit_code, atomizer_output)
internal_errors.labels(ErrorType.AtomizerFailure).inc() # pylint: disable=no-member
raise AtomizerError('Atomizer command failed!')
new_atoms = []
for atom_value in atomizer_output.strip().splitlines():
# For purposes of matching atom string values across builds, we must replace the generated/unique
# project directory with its corresponding universal environment variable: '$PROJECT_DIR'.
atom_value = atom_value.replace(project_type.project_directory, '$PROJECT_DIR')
new_atoms.append(Atom(get_environment_variable_setter_command(atomizer_var_name, atom_value)))
atoms_list.extend(new_atoms)
return atoms_list
def _handle_setup_failure_on_slave(self, slave):
"""
Respond to failed build setup on a slave. This should put the slave back into a usable state.
:type slave: Slave
"""
internal_errors.labels(ErrorType.SetupBuildFailure).inc() # pylint: disable=no-member
raise BadRequestError('Setup failure handling on the master is not yet implemented.')
def _request(self,
method,
url,
data=None,
should_encode_body=True,
error_on_failure=False,
timeout=None,
*args,
**kwargs):
"""
A wrapper around requests library network request methods (e.g., GET, POST). We can add functionality for
unimplemented request methods as needed. We also do some mutation on request bodies to make receiving data (in
the Tornado request handlers) a bit more convenient.
:param method: The request method -- passed through to requests lib
:type method: str
:param url: The request url -- passed through to requests lib
:type url: str
:param data: The request body data -- passed through to requests lib but may be mutated first
:type data: dict|str|bytes|None
:param should_encode_body: If True we json-encode the actual data inside a new dict, which is then sent with
the request. This should be True for all Master<->Slave API calls but we can set this False to skip this
extra encoding step if necessary.
:type should_encode_body: bool
:param error_on_failure: If true, raise an error when the response is not in the 200s
:type error_on_failure: bool
:param timeout: The timeout in seconds for this request; raises requests.Timeout upon timeout.
:type timeout: int|None
:rtype: requests.Response
"""
timeout = timeout or Configuration['default_http_timeout']
# If data type is dict, we json-encode it and nest the encoded string inside a new dict. This prevents the
# requests library from trying to directly urlencode the key value pairs of the original data, which will
# not correctly encode anything in a nested dict. The inverse of this encoding is done on the receiving end in
# ClusterBaseHandler. The reason we nest this in another dict instead of sending the json string directly is
# that if files are included in a post request, the type of the data argument *must* be a dict.
data_to_send = data
if should_encode_body and isinstance(data_to_send, dict):
data_to_send = {ENCODED_BODY: self.encode_body(data_to_send)}
resp = self._session.request(method,
url,
data=data_to_send,
timeout=timeout,
*args,
**kwargs)
if not resp.ok and error_on_failure:
internal_errors.labels(ErrorType.NetworkRequestFailure).inc() # pylint: disable=no-member
raise RequestFailedError(
'Request to {} failed with status_code {} and response "{}"'.
format(url, str(resp.status_code), resp.text))
return resp
def _request(
self,
method,
url,
data=None,
should_encode_body=True,
error_on_failure=False,
timeout=None,
*args,
**kwargs
):
"""
A wrapper around requests library network request methods (e.g., GET, POST). We can add functionality for
unimplemented request methods as needed. We also do some mutation on request bodies to make receiving data (in
the Tornado request handlers) a bit more convenient.
:param method: The request method -- passed through to requests lib
:type method: str
:param url: The request url -- passed through to requests lib
:type url: str
:param data: The request body data -- passed through to requests lib but may be mutated first
:type data: dict|str|bytes|None
:param should_encode_body: If True we json-encode the actual data inside a new dict, which is then sent with
the request. This should be True for all Master<->Slave API calls but we can set this False to skip this
extra encoding step if necessary.
:type should_encode_body: bool
:param error_on_failure: If true, raise an error when the response is not in the 200s
:type error_on_failure: bool
:param timeout: The timeout in seconds for this request; raises requests.Timeout upon timeout.
:type timeout: int|None
:rtype: requests.Response
"""
timeout = timeout or Configuration['default_http_timeout']
# If data type is dict, we json-encode it and nest the encoded string inside a new dict. This prevents the
# requests library from trying to directly urlencode the key value pairs of the original data, which will
# not correctly encode anything in a nested dict. The inverse of this encoding is done on the receiving end in
# ClusterBaseHandler. The reason we nest this in another dict instead of sending the json string directly is
# that if files are included in a post request, the type of the data argument *must* be a dict.
data_to_send = data
if should_encode_body and isinstance(data_to_send, dict):
data_to_send = {ENCODED_BODY: self.encode_body(data_to_send)}
resp = self._session.request(method, url, data=data_to_send, timeout=timeout, *args, **kwargs)
if not resp.ok and error_on_failure:
internal_errors.labels(ErrorType.NetworkRequestFailure).inc() # pylint: disable=no-member
raise RequestFailedError('Request to {} failed with status_code {} and response "{}"'.
format(url, str(resp.status_code), resp.text))
return resp
def _handle_setup_failure_on_slave(self, slave):
"""
Respond to failed build setup on a slave. This should put the slave back into a usable state.
:type slave: Slave
"""
internal_errors.labels(ErrorType.SetupBuildFailure).inc() # pylint: disable=no-member
build = self.get_build(slave.current_build_id)
build.setup_failures += 1
if build.setup_failures >= MAX_SETUP_FAILURES:
build.cancel()
build.mark_failed(
'Setup failed on this build more than {} times. Failing the build.'
.format(MAX_SETUP_FAILURES))
slave.teardown()
def _handle_subjob_payload(self, subjob_id, payload):
if not payload:
self._logger.warning('No payload for subjob {} of build {}.', subjob_id, self._build_id)
return
# Assertion: all payloads received from subjobs are uniquely named.
result_file_path = os.path.join(self._build_results_dir(), payload['filename'])
try:
app.util.fs.write_file(payload['body'], result_file_path)
app.util.fs.extract_tar(result_file_path, delete=True)
self._parse_payload_for_atom_exit_code(subjob_id)
except:
internal_errors.labels(ErrorType.SubjobWriteFailure).inc() # pylint: disable=no-member
self._logger.warning('Writing payload for subjob {} of build {} FAILED.', subjob_id, self._build_id)
raise
def _perform_async_postbuild_tasks(self):
"""
Once a build is complete, execute certain tasks like archiving the artifacts and writing timing
data. This method also transitions the FSM to finished after the postbuild tasks are complete.
"""
try:
timing_data = self._read_subjob_timings_from_results()
self._create_build_artifact(timing_data)
serialized_build_time_seconds.observe(sum(timing_data.values()))
self._delete_temporary_build_artifact_files()
self._postbuild_tasks_are_finished = True
self._state_machine.trigger(BuildEvent.POSTBUILD_TASKS_COMPLETE)
self._logger.notice('Completed build (id: {}), saving to database.'.format(self._build_id))
self.save()
except Exception as ex: # pylint: disable=broad-except
internal_errors.labels(ErrorType.PostBuildFailure).inc() # pylint: disable=no-member
self._logger.exception('Postbuild tasks failed for build {}.'.format(self._build_id))
self.mark_failed('Postbuild tasks failed due to an internal error: "{}"'.format(ex))
def _create_build_artifact(self, timing_data: Dict[str, float]): # pylint: disable=unsubscriptable-object
self._build_artifact = BuildArtifact(self._build_results_dir())
self._build_artifact.generate_failures_file()
self._build_artifact.write_timing_data(self._timing_file_path, timing_data)
self._artifacts_tar_file = app.util.fs.tar_directory(self._build_results_dir(),
BuildArtifact.ARTIFACT_TARFILE_NAME)
temp_tar_path = None
try:
# Temporarily move aside tar file so we can create a zip file, then move it back.
# This juggling can be removed once we're no longer creating tar artifacts.
temp_tar_path = shutil.move(self._artifacts_tar_file, tempfile.mktemp())
self._artifacts_zip_file = app.util.fs.zip_directory(self._build_results_dir(),
BuildArtifact.ARTIFACT_ZIPFILE_NAME)
except Exception: # pylint: disable=broad-except
internal_errors.labels(ErrorType.ZipFileCreationFailure).inc() # pylint: disable=no-member
# Due to issue #339 we are ignoring exceptions in the zip file creation for now.
self._logger.exception('Zipping of artifacts failed. This error will be ignored.')
finally:
if temp_tar_path:
shutil.move(temp_tar_path, self._artifacts_tar_file)
def atomize_in_project(self, project_type):
"""
Translate the atomizer dicts that this instance was initialized with into a list of actual atom commands. This
executes atomizer commands inside the given project in order to generate the atoms.
:param project_type: The ProjectType instance in which to execute the atomizer commands
:type project_type: ProjectType
:return: The list of environment variable "export" atom commands
:rtype: list[app.master.atom.Atom]
"""
atoms_list = []
for atomizer_dict in self._atomizer_dicts:
for atomizer_var_name, atomizer_command in atomizer_dict.items():
atomizer_output, exit_code = project_type.execute_command_in_project(
atomizer_command)
if exit_code != 0:
self._logger.error(
'Atomizer command "{}" for variable "{}" failed with exit code: {} and output:'
'\n{}', atomizer_command, atomizer_var_name, exit_code,
atomizer_output)
internal_errors.labels(ErrorType.AtomizerFailure).inc() # pylint: disable=no-member
raise AtomizerError('Atomizer command failed!')
new_atoms = []
for atom_value in atomizer_output.strip().splitlines():
# For purposes of matching atom string values across builds, we must replace the generated/unique
# project directory with its corresponding universal environment variable: '$PROJECT_DIR'.
atom_value = atom_value.replace(
project_type.project_directory, '$PROJECT_DIR')
new_atoms.append(
Atom(
get_environment_variable_setter_command(
atomizer_var_name, atom_value)))
atoms_list.extend(new_atoms)
return atoms_list