def get_slave(self, slave_id=None, slave_url=None):
"""
Get the instance of given slave by either the slave's id or url. Only one of slave_id or slave_url should be
specified.
:param slave_id: The id of the slave to return
:type slave_id: int
:param slave_url: The url of the slave to return
:type slave_url: str
:return: The instance of the slave
:rtype: Slave
"""
if (slave_id is None) == (slave_url is None):
raise ValueError(
'Only one of slave_id or slave_url should be specified to get_slave().'
)
if slave_id is not None:
for slave in self._all_slaves_by_url.values():
if slave.id == slave_id:
return slave
else:
if slave_url in self._all_slaves_by_url:
return self._all_slaves_by_url[slave_url]
raise ItemNotFoundError(
'Requested slave ({}) does not exist.'.format(slave_id))
def subjob(self, subjob_id):
"""
Returns a single subjob
:type subjob_id: int
:rtype: Subjob
"""
subjob = self._all_subjobs_by_id.get(subjob_id)
if subjob is None:
raise ItemNotFoundError('Invalid subjob id.')
return subjob
def get(cls, build_id: int) -> Build:
"""
Returns a build by id
:param build_id: The id for the build whose status we are getting
"""
build = cls._all_builds_by_id.get(build_id)
if build is None:
raise ItemNotFoundError('Invalid build id: {}.'.format(build_id))
return build
def get_build(self, build_id):
"""
Returns a build by id
:param build_id: The id for the build whose status we are getting
:type build_id: int
:rtype: Build
"""
build = self._all_builds_by_id.get(build_id)
if build is None:
raise ItemNotFoundError('Invalid build id.')
return build
def get_build(self, build_id):
"""
Returns a build by id
:param build_id: The id for the build whose status we are getting
:type build_id: int
:rtype: Build
"""
build = BuildStore.get(build_id)
if build is None:
raise ItemNotFoundError('Invalid build id: {}.'.format(build_id))
return build
def get_console_output(
self,
build_id: int,
subjob_id: int,
atom_id: int,
result_root: str,
max_lines: int = 50,
offset_line: Optional[int] = None,
):
"""
Return the console output if it exists, raises an ItemNotFound error if not.
On success, the response contains keys: offset_line, num_lines, total_num_lines, and content.
e.g.:
{
'offset_line': 0,
'num_lines': 50,
'total_num_lines': 167,
'content': 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit,\n...',
}
:param build_id: build id
:param subjob_id: subjob id
:param atom_id: atom id
:param result_root: the sys path to either the results or artifacts directory where results are stored.
:param max_lines: The maximum total number of lines to return. If this max_lines + offset_line lines do not
exist in the output file, just return what there is.
:param offset_line: The line number (0-indexed) to start reading content for. If none is specified, we will
return the console output starting from the end of the file.
"""
if offset_line is not None and offset_line < 0:
raise BadRequestError(
'\'offset_line\' must be greater than or equal to zero.')
if max_lines <= 0:
raise BadRequestError('\'max_lines\' must be greater than zero.')
segment = BuildArtifact.get_console_output(build_id, subjob_id,
atom_id, result_root,
max_lines, offset_line)
if not segment:
raise ItemNotFoundError(
'Console output does not exist on this host for '
'build {}, subjob {}, atom {}.'.format(build_id, subjob_id,
atom_id))
return {
'offset_line': segment.offset_line,
'num_lines': segment.num_lines,
'total_num_lines': segment.total_num_lines,
'content': segment.content,
}
def get_console_output(self, build_id, subjob_id, atom_id, result_root, max_lines=50, offset_line=None):
"""
Return the console output if it exists, raises an ItemNotFound error if not.
On success, the response contains keys: offset_line, num_lines, total_num_lines, and content.
e.g.:
{
'offset_line': 0,
'num_lines': 50,
'total_num_lines': 167,
'content': 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit,\n...',
}
:type build_id: int
:type subjob_id: int
:type atom_id: int
:param result_root: the sys path to either the results or artifacts directory where results are stored.
:type result_root: str
:param max_lines: The maximum total number of lines to return. If this max_lines + offset_line lines do not
exist in the output file, just return what there is.
:type max_lines: int
:param offset_line: The line number (0-indexed) to start reading content for. If none is specified, we will
return the console output starting from the end of the file.
:type offset_line: int | None
"""
if offset_line is not None and offset_line < 0:
raise BadRequestError('\'offset_line\' must be greater than or equal to zero.')
if max_lines <= 0:
raise BadRequestError('\'max_lines\' must be greater than zero.')
artifact_dir = BuildArtifact.atom_artifact_directory(build_id, subjob_id, atom_id, result_root=result_root)
output_file = os.path.join(artifact_dir, BuildArtifact.OUTPUT_FILE)
if not os.path.isfile(output_file):
raise ItemNotFoundError('Output file doesn\'t exist for build_id: {} subjob_id: {} atom_id: {}'.format(
build_id, subjob_id, atom_id))
try:
console_output = ConsoleOutput(output_file)
segment = console_output.segment(max_lines, offset_line)
except ValueError as e:
raise BadRequestError(e)
return {
'offset_line': segment.offset_line,
'num_lines': segment.num_lines,
'total_num_lines': segment.total_num_lines,
'content': segment.content,
}
def get_path_for_build_results_archive(self, build_id: int, is_tar_request: bool=False) -> str:
"""
Given a build id, get the absolute file path for the archive file containing the build results.
:param build_id: The build id for which to retrieve the artifacts archive file
:param is_tar_request: If true, download the tar.gz archive instead of a zip.
:return: The path to the archived results file
"""
build = self._all_builds_by_id.get(build_id) # type: Build
if build is None:
raise ItemNotFoundError('Invalid build id.')
archive_file = build.artifacts_tar_file if is_tar_request else build.artifacts_zip_file
if archive_file is None:
raise ItemNotReadyError('Build artifact file is not yet ready. Try again later.')
return archive_file
def handle_request_to_update_build(self, build_id, update_params):
"""
Updates the state of a build with the values passed in. Used for cancelling running builds.
:type build_id: int
:param update_params: The fields that should be updated and their new values
:type update_params: dict [str, str]
:return: The success/failure and the response we want to send to the requestor
:rtype: tuple [bool, dict [str, str]]
"""
build = self._all_builds_by_id.get(int(build_id))
if build is None:
raise ItemNotFoundError('Invalid build id.')
success, response = build.validate_update_params(update_params)
if not success:
return success, response
return build.update_state(update_params), {}
def get_path_for_build_results_archive(self, build_id):
"""
Given a build id, get the absolute file path for the archive file containing the build results.
:param build_id: The build id for which to retrieve the artifacts archive file
:type build_id: int
:return: The path to the archived results file
:rtype: str
"""
build = self._all_builds_by_id.get(build_id)
if build is None:
raise ItemNotFoundError('Invalid build id.')
archive_file = build.artifacts_archive_file
if archive_file is None:
raise ItemNotReadyError('Build artifact file is not yet ready. Try again later.')
return archive_file
def get_slave(self, slave_id: int=None, slave_url: str=None) -> Slave:
"""
Look for a slave in the registry and if not found raise "ItemNotFoundError" exception.
:raises ItemNotFoundError when slave does not exists in Registry.
"""
if (slave_id is None) == (slave_url is None):
raise ValueError('Only one of slave_id or slave_url should be specified to get_slave().')
if slave_id is not None:
if slave_id in self._all_slaves_by_id:
return self._all_slaves_by_id[slave_id]
else:
if slave_url in self._all_slaves_by_url:
return self._all_slaves_by_url[slave_url]
if slave_id is not None:
error_msg = 'Requested slave (slave_id={}) does not exist.'.format(slave_id)
else:
error_msg = 'Requested slave (slave_url={}) does not exist.'.format(slave_url)
raise ItemNotFoundError(error_msg)
def save(self):
"""Serialize the Build object and update all of the parts to the database."""
with Connection.get() as session:
build_schema = session.query(BuildSchema).filter(BuildSchema.build_id == self._build_id).first()
failed_artifact_directories_schema = session.query(FailedArtifactDirectoriesSchema) \
.filter(FailedArtifactDirectoriesSchema.build_id == self._build_id) \
.all()
failed_subjob_atom_pairs_schema = session.query(FailedSubjobAtomPairsSchema) \
.filter(FailedSubjobAtomPairsSchema.build_id == self._build_id) \
.all()
atoms_schema = session.query(AtomsSchema).filter(AtomsSchema.build_id == self._build_id).all()
subjobs_schema = session.query(SubjobsSchema).filter(SubjobsSchema.build_id == self._build_id).all()
# If this wasn't found, it's safe to assume that the build doesn't exist within the database
if build_schema is None:
raise ItemNotFoundError('Unable to find build (id: {}) in database.'.format(self._build_id))
build_schema.artifacts_tar_file = self._artifacts_tar_file
build_schema.artifacts_zip_file = self._artifacts_zip_file
build_schema.error_message = self._error_message
build_schema.postbuild_tasks_are_finished = self._postbuild_tasks_are_finished
build_schema.setup_failures = self.setup_failures
build_schema.timing_file_path = self._timing_file_path
build_artifact_dir = None
if self._build_artifact is not None:
build_artifact_dir = self._build_artifact.build_artifact_dir
build_schema.build_artifact_dir = build_artifact_dir
if self._build_artifact is not None:
# Clear all old directories
session.query(FailedArtifactDirectoriesSchema) \
.filter(FailedArtifactDirectoriesSchema.build_id == self._build_id) \
.delete()
# Commit changes so we don't delete the newly added rows later
session.commit()
# Add all the updated versions of the directories
for directory in self._build_artifact._get_failed_artifact_directories():
failed_artifact_directory = FailedArtifactDirectoriesSchema(
build_id=self._build_id,
failed_artifact_directory=directory
)
session.add(failed_artifact_directory)
if self._build_artifact is not None:
# Clear all old directories
session.query(FailedSubjobAtomPairsSchema) \
.filter(FailedSubjobAtomPairsSchema.build_id == self._build_id) \
.delete()
# Commit changes so we don't delete the newly added rows later
session.commit()
# Add all the updated versions of the data
for subjob_id, atom_id in self._build_artifact.get_failed_subjob_and_atom_ids():
failed_subjob_and_atom_ids = FailedSubjobAtomPairsSchema(
build_id=self._build_id,
subjob_id=subjob_id,
atom_id=atom_id
)
session.add(failed_subjob_and_atom_ids)
build_schema.build_parameters = json.dumps(self._build_request.build_parameters())
fsm_timestamps = {state.lower(): timestamp for state, timestamp in self._state_machine.transition_timestamps.items()}
build_schema.state = self._status()
build_schema.queued_ts = fsm_timestamps['queued']
build_schema.finished_ts = fsm_timestamps['finished']
build_schema.prepared_ts = fsm_timestamps['prepared']
build_schema.preparing_ts = fsm_timestamps['preparing']
build_schema.error_ts = fsm_timestamps['error']
build_schema.canceled_ts = fsm_timestamps['canceled']
build_schema.building_ts = fsm_timestamps['building']
# Subjobs
# Clear all old Subjobs and Atoms
session.query(SubjobsSchema) \
.filter(SubjobsSchema.build_id == self._build_id) \
.delete()
session.query(AtomsSchema) \
.filter(AtomsSchema.build_id == self._build_id) \
.delete()
# Commit changes so we don't delete the newly added rows later
session.commit()
# Add all the updated versions of Subjobs and Atoms
subjobs = self._all_subjobs_by_id
for subjob_id in subjobs:
subjob = self._all_subjobs_by_id[subjob_id]
subjob_schema = SubjobsSchema(
subjob_id=subjob_id,
build_id=self._build_id,
completed=subjob.completed
)
session.add(subjob_schema)
# Atoms
for atom in subjob._atoms:
atom_schema = AtomsSchema(
atom_id=atom.id,
build_id=self._build_id,
subjob_id=subjob_id,
command_string=atom.command_string,
expected_time=atom.expected_time,
actual_time=atom.actual_time,
exit_code=atom.exit_code,
state=atom.state
)
session.add(atom_schema)
def subjob(self, subjob_id: int) -> Subjob:
"""Return the subjob for this build with the specified id."""
subjob = self._all_subjobs_by_id.get(subjob_id)
if subjob is None:
raise ItemNotFoundError('Invalid subjob id.')
return subjob