class Compose(FreshmakerBase):
__tablename__ = 'composes'
id = db.Column(db.Integer, primary_key=True)
odcs_compose_id = db.Column(db.Integer, nullable=False)
builds = db.relationship('ArtifactBuildCompose', back_populates='compose')
@property
def finished(self):
from freshmaker.odcsclient import create_odcs_client
return 'done' == create_odcs_client().get_compose(
self.odcs_compose_id)['state_name']
@classmethod
def get_lowest_compose_id(cls, session):
"""
Returns the lowest odcs_compose_id. If there is no compose,
returns 0.
"""
compose = session.query(Compose).order_by(
Compose.odcs_compose_id.asc()).first()
if not compose:
return 0
return compose.odcs_compose_id
class User(FreshmakerBase, UserMixin):
"""User information table"""
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(200), nullable=False, unique=True)
@classmethod
def find_user_by_name(cls, username):
"""Find a user by username
:param str username: a string of username to find user
:return: user object if found, otherwise None is returned.
:rtype: User
"""
try:
return db.session.query(cls).filter(cls.username == username)[0]
except IndexError:
return None
@classmethod
def create_user(cls, username):
user = cls(username=username)
db.session.add(user)
return user
class EventDependency(FreshmakerBase):
__tablename__ = "event_dependencies"
id = db.Column(db.Integer, primary_key=True)
event_id = db.Column(db.Integer,
db.ForeignKey('events.id'),
nullable=False)
event_dependency_id = db.Column(db.Integer,
db.ForeignKey('events.id'),
nullable=False)
class ArtifactBuildCompose(FreshmakerBase):
__tablename__ = 'artifact_build_composes'
build_id = db.Column(db.Integer,
db.ForeignKey('artifact_builds.id'),
primary_key=True)
compose_id = db.Column(db.Integer,
db.ForeignKey('composes.id'),
primary_key=True)
build = db.relationship('ArtifactBuild', back_populates='composes')
compose = db.relationship('Compose', back_populates='builds')
class ArtifactBuild(FreshmakerBase):
__tablename__ = "artifact_builds"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, nullable=False)
original_nvr = db.Column(db.String, nullable=True)
rebuilt_nvr = db.Column(db.String, nullable=True)
type = db.Column(db.Integer)
state = db.Column(db.Integer, nullable=False)
state_reason = db.Column(db.String, nullable=True)
time_submitted = db.Column(db.DateTime, nullable=False)
time_completed = db.Column(db.DateTime)
# Link to the Artifact on which this one depends and which triggered
# the rebuild of this Artifact.
dep_on_id = db.Column(db.Integer, db.ForeignKey('artifact_builds.id'))
dep_on = relationship('ArtifactBuild', remote_side=[id])
# Event associated with this Build
event_id = db.Column(db.Integer, db.ForeignKey('events.id'))
event = relationship("Event", back_populates="builds")
# Id of corresponding real build in external build system. Currently, it
# could be ID of a build in MBS or Koji, maybe others in the future.
# build_id may be NULL, which means this build has not been built in
# external build system.
build_id = db.Column(db.Integer)
# Build args in json format.
build_args = db.Column(db.String, nullable=True)
# The reason why this artifact is rebuilt. Set according to
# `freshmaker.types.RebuildReason`.
rebuild_reason = db.Column(db.Integer, nullable=True)
# pullspec overrides
_bundle_pullspec_overrides = db.Column("bundle_pullspec_overrides",
db.Text,
nullable=True)
composes = db.relationship('ArtifactBuildCompose', back_populates='build')
@classmethod
def create(cls,
session,
event,
name,
type,
build_id=None,
dep_on=None,
state=None,
original_nvr=None,
rebuilt_nvr=None,
rebuild_reason=0):
now = datetime.utcnow()
build = cls(name=name,
original_nvr=original_nvr,
rebuilt_nvr=rebuilt_nvr,
type=type,
event=event,
state=state or ArtifactBuildState.BUILD.value,
build_id=build_id,
time_submitted=now,
dep_on=dep_on,
rebuild_reason=rebuild_reason)
session.add(build)
return build
@validates('state')
def validate_state(self, key, field):
if field in [s.value for s in list(ArtifactBuildState)]:
return field
if field in [s.name.lower() for s in list(ArtifactBuildState)]:
return ArtifactBuildState[field.upper()].value
if isinstance(field, ArtifactBuildState):
return field.value
raise ValueError("%s: %s, not in %r" %
(key, field, list(ArtifactBuildState)))
@validates('type')
def validate_type(self, key, field):
if field in [t.value for t in list(ArtifactType)]:
return field
if field in [t.name.lower() for t in list(ArtifactType)]:
return ArtifactType[field.upper()].value
if isinstance(field, ArtifactType):
return field.value
raise ValueError("%s: %s, not in %r" %
(key, field, list(ArtifactType)))
@classmethod
def get_lowest_build_id(cls, session):
"""
Returns the lowest build_id. If there is no build so far,
returns 0.
"""
build = (
session.query(ArtifactBuild).filter(cls.build_id != None) # noqa
.order_by(ArtifactBuild.build_id.asc()).first())
if not build:
return 0
return build.build_id
@property
def bundle_pullspec_overrides(self):
"""Return the Python representation of the JSON bundle_pullspec_overrides."""
return (json.loads(self._bundle_pullspec_overrides)
if self._bundle_pullspec_overrides else None)
@bundle_pullspec_overrides.setter
def bundle_pullspec_overrides(self, bundle_pullspec_overrides):
"""
Set the bundle_pullspec_overrides column to the input bundle_pullspec_overrides as a JSON string.
If ``None`` is provided, it will be simply set to ``None`` and not be converted to JSON.
:param dict bundle_pullspec_overrides: the dictionary of the bundle_pullspec_overrides or ``None``
"""
self._bundle_pullspec_overrides = (json.dumps(
bundle_pullspec_overrides,
sort_keys=True) if bundle_pullspec_overrides is not None else None)
def depending_artifact_builds(self):
"""
Returns list of artifact builds depending on this one.
"""
return ArtifactBuild.query.filter_by(dep_on_id=self.id).all()
def transition(self, state, state_reason):
"""
Sets the state and state_reason of this ArtifactBuild.
:param state: ArtifactBuildState value
:param state_reason: Reason why this state has been set.
:return: True/False, whether state was changed
"""
# Convert state from its possible representation to number.
state = self.validate_state("state", state)
# Log the state and state_reason
if state == ArtifactBuildState.FAILED.value:
log_fnc = log.error
else:
log_fnc = log.info
log_fnc("Artifact build %r moved to state %s, %r" %
(self, ArtifactBuildState(state).name, state_reason))
if self.state == state:
return False
self.state = state
if ArtifactBuildState(state).counter:
ArtifactBuildState(state).counter.inc()
self.state_reason = state_reason
if self.state in [
ArtifactBuildState.DONE.value, ArtifactBuildState.FAILED.value,
ArtifactBuildState.CANCELED.value
]:
self.time_completed = datetime.utcnow()
# For FAILED/CANCELED states, move also all the artifacts depending
# on this one to FAILED/CANCELED state, because there is no way we
# can rebuild them.
if self.state in [
ArtifactBuildState.FAILED.value,
ArtifactBuildState.CANCELED.value
]:
for build in self.depending_artifact_builds():
build.transition(
self.state, "Cannot build artifact, because its "
"dependency cannot be built.")
messaging.publish('build.state.changed', self.json())
return True
def __repr__(self):
return "<ArtifactBuild %s, type %s, state %s, event %s>" % (
self.name, ArtifactType(self.type).name,
ArtifactBuildState(self.state).name, self.event.message_id)
def json(self):
build_args = {}
if self.build_args:
build_args = json.loads(self.build_args)
build_url = get_url_for('build', id=self.id)
db.session.add(self)
return {
"id": self.id,
"name": self.name,
"original_nvr": self.original_nvr,
"rebuilt_nvr": self.rebuilt_nvr,
"type": self.type,
"type_name": ArtifactType(self.type).name,
"state": self.state,
"state_name": ArtifactBuildState(self.state).name,
"state_reason": self.state_reason,
"dep_on": self.dep_on.name if self.dep_on else None,
"dep_on_id": self.dep_on.id if self.dep_on else None,
"time_submitted": _utc_datetime_to_iso(self.time_submitted),
"time_completed": _utc_datetime_to_iso(self.time_completed),
"event_id": self.event_id,
"build_id": self.build_id,
"url": build_url,
"build_args": build_args,
"odcs_composes":
[rel.compose.odcs_compose_id for rel in self.composes],
"rebuild_reason": RebuildReason(self.rebuild_reason
or 0).name.lower()
}
def get_root_dep_on(self):
dep_on = self.dep_on
while dep_on:
dep = dep_on.dep_on
if dep:
dep_on = dep
else:
break
return dep_on
def add_composes(self, session, composes):
"""Add an ODCS compose to this build"""
for compose in composes:
session.add(
ArtifactBuildCompose(build_id=self.id, compose_id=compose.id))
@property
def composes_ready(self):
"""Check if composes this build has have been done in ODCS"""
return all((rel.compose.finished for rel in self.composes))
class Event(FreshmakerBase):
__tablename__ = "events"
id = db.Column(db.Integer, primary_key=True)
# ID of message generating the rebuild event.
message_id = db.Column(db.String, nullable=False)
# Searchable key for the event - used when searching for events from the JSON
# API.
search_key = db.Column(db.String, nullable=False)
# Event type id defined in EVENT_TYPES - ID of class inherited from
# BaseEvent class - used when searching for events of particular type.
event_type_id = db.Column(db.Integer, nullable=False)
# True when the Event is already released and we do not have to include
# it in the future rebuilds of artifacts.
# This is currently only used for internal Docker images rebuilds, but in
# the future might be used even for modules or Fedora Docker images.
released = db.Column(db.Boolean, default=True)
state = db.Column(db.Integer, nullable=False)
state_reason = db.Column(db.String, nullable=True)
time_created = db.Column(db.DateTime, nullable=True)
time_done = db.Column(db.DateTime, nullable=True)
# AppenderQuery for getting builds associated with this Event.
builds = relationship("ArtifactBuild",
back_populates="event",
lazy="dynamic",
cascade="all, delete-orphan",
passive_deletes=True)
# True if the even should be handled in dry run mode.
dry_run = db.Column(db.Boolean, default=False)
# For manual rebuilds, set to user requesting the rebuild. Otherwise null.
requester = db.Column(db.String, nullable=True)
# For manual rebuilds, contains the white-space separate list of artifacts
# (for example NVR of container images) to rebuild if passed using the
# REST API.
requested_rebuilds = db.Column(db.String, nullable=True)
# For manual rebuilds, contains the serialized JSON optionally submitted
# by the requester to track the context of this event.
requester_metadata = db.Column(db.String, nullable=True)
manual_triggered = db.Column(
db.Boolean,
default=False,
doc='Whether this event is triggered manually')
@classmethod
def create(cls,
session,
message_id,
search_key,
event_type,
released=True,
state=None,
manual=False,
dry_run=False,
requester=None,
requested_rebuilds=None,
requester_metadata=None):
if event_type in EVENT_TYPES:
event_type = EVENT_TYPES[event_type]
now = datetime.utcnow()
event = cls(
message_id=message_id,
search_key=search_key,
event_type_id=event_type,
released=released,
state=state or EventState.INITIALIZED.value,
time_created=now,
manual_triggered=manual,
dry_run=dry_run,
requester=requester,
requested_rebuilds=requested_rebuilds,
requester_metadata=requester_metadata,
)
session.add(event)
return event
@validates('state')
def validate_state(self, key, field):
if field in [s.value for s in list(EventState)]:
return field
if field in [s.name.lower() for s in list(EventState)]:
return EventState[field.upper()].value
if isinstance(field, EventState):
return field.value
raise ValueError("%s: %s, not in %r" % (key, field, list(EventState)))
@classmethod
def get(cls, session, message_id):
return session.query(cls).filter_by(message_id=message_id).first()
@classmethod
def get_or_create(cls,
session,
message_id,
search_key,
event_type,
released=True,
manual=False,
dry_run=False,
requester=None,
requested_rebuilds=None,
requester_metadata=None):
instance = cls.get(session, message_id)
if instance:
return instance
instance = cls.create(session,
message_id,
search_key,
event_type,
released=released,
manual=manual,
dry_run=dry_run,
requester=requester,
requested_rebuilds=requested_rebuilds,
requester_metadata=requester_metadata)
session.commit()
return instance
@classmethod
def get_or_create_from_event(cls, session, event, released=True):
# we must extract all needed arguments,
# because event might not have some of them so we will use defaults
requester = getattr(event, "requester", None)
requested_rebuilds_list = getattr(event, "container_images", None)
requested_rebuilds = None
# make sure 'container_images' field is a list and convert it to str
if requested_rebuilds_list is not None and \
isinstance(requested_rebuilds_list, list):
requested_rebuilds = " ".join(requested_rebuilds_list)
requester_metadata = getattr(event, "requester_metadata_json", None)
if requester_metadata is not None:
# try to convert JSON into str, if it's invalid use None
try:
requester_metadata = json.dumps(requester_metadata)
except TypeError:
log.warning(
"requester_metadata_json field is ill-formatted: %s",
requester_metadata)
requester_metadata = None
return cls.get_or_create(session,
event.msg_id,
event.search_key,
event.__class__,
released=released,
manual=event.manual,
dry_run=event.dry_run,
requester=requester,
requested_rebuilds=requested_rebuilds,
requester_metadata=requester_metadata)
@classmethod
def get_unreleased(cls, session, states=None):
"""
Returns list of all unreleased events in given states. If no states
are provided, returns only events in INITIALIZED, BUILDING or COMPLETE
state.
:param session: db.session
:param list states: List of states to filter events for. If None,
INITIALIZED, BUILDING and COMPLETE is used.
:rtype: list of models.Event.
:return: List of unreleased events of `states` state.
"""
if not states:
states = [
EventState.INITIALIZED.value, EventState.BUILDING.value,
EventState.COMPLETE.value
]
else:
states = [
state.value if isinstance(state, EventState) else state
for state in states
]
return session.query(cls).filter(cls.released == false(),
cls.state.in_(states)).all()
@classmethod
def get_by_event_id(cls, session, event_id):
return session.query(cls).filter_by(id=event_id).first()
def get_image_builds_in_first_batch(self, session):
return session.query(ArtifactBuild).filter_by(
dep_on=None,
type=ArtifactType.IMAGE.value,
event_id=self.id,
).all()
@property
def event_type(self):
return INVERSE_EVENT_TYPES[self.event_type_id]
def add_event_dependency(self, session, event):
"""Add a dependent event
:param session: the `db.session`.
:param event: the dependent event to be added.
:type event: :py:class:`Event`
:return: instance of :py:class:`EventDependency`. Caller is responsible
for committing changes to database. If `event` has been added
already, nothing changed and `None` will be returned.
"""
dep = session.query(EventDependency.id).filter_by(
event_id=self.id, event_dependency_id=event.id).first()
if dep is None:
dep = EventDependency(event_id=self.id,
event_dependency_id=event.id)
session.add(dep)
return dep
else:
return None
@property
def event_dependencies(self):
"""
Returns the list of Events this Event depends on.
"""
events = []
deps = EventDependency.query.filter_by(event_id=self.id).all()
for dep in deps:
events.append(
Event.query.filter_by(id=dep.event_dependency_id).first())
return events
@property
def depending_events(self):
"""
Returns the list of Events depending on this Event.
"""
depending_events = []
parents = EventDependency.query.filter_by(
event_dependency_id=self.id).all()
for p in parents:
depending_events.append(
Event.query.filter_by(id=p.event_id).first())
return depending_events
def has_all_builds_in_state(self, state):
"""
Returns True when all builds are in the given `state`.
"""
return db.session.query(ArtifactBuild).filter_by(
event_id=self.id).filter(state != state).count() == 0
def builds_transition(self, state, reason, filters=None):
"""
Calls transition(state, reason) for all builds associated with this
event.
:param dict filters: Filter only specific builds to transition.
:return: list of build ids which were transitioned
"""
if not self.builds:
return []
builds_to_transition = self.builds.filter_by(
**filters).all() if isinstance(filters, dict) else self.builds
return [
build.id for build in builds_to_transition
if build.transition(state, reason)
]
def transition(self, state, state_reason=None):
"""
Sets the time_done, state, and state_reason of this Event.
:param state: EventState value
:param state_reason: Reason why this state has been set.
:return: True/False, whether state was changed
"""
# Convert state from its possible representation to number.
state = self.validate_state("state", state)
# Update the state reason.
if state_reason is not None:
self.state_reason = state_reason
# Log the state and state_reason
if state == EventState.FAILED.value:
log_fnc = log.error
else:
log_fnc = log.info
log_fnc("Event %r moved to state %s, %r" %
(self, EventState(state).name, state_reason))
# In case Event is already in the state, return False.
if self.state == state:
return False
self.state = state
# Log the time done
if state in [
EventState.FAILED.value, EventState.COMPLETE.value,
EventState.SKIPPED.value, EventState.CANCELED.value
]:
self.time_done = datetime.utcnow()
if EventState(state).counter:
EventState(state).counter.inc()
db.session.commit()
messaging.publish('event.state.changed', self.json())
messaging.publish('event.state.changed.min', self.json_min())
return True
def __repr__(self):
return "<Event %s, %r, %s>" % (self.message_id, self.event_type,
self.search_key)
def __str__(self):
if self.event_type_id in INVERSE_EVENT_TYPES:
type_name = INVERSE_EVENT_TYPES[self.event_type_id].__name__
else:
type_name = "UnknownEventType %d" % self.event_type_id
return "<%s, search_key=%s>" % (type_name, self.search_key)
@property
def requester_metadata_json(self):
if not self.requester_metadata:
return {}
return json.loads(self.requester_metadata)
def json(self):
data = self._common_json()
data['builds'] = [b.json() for b in self.builds]
return data
def json_min(self):
builds_summary = defaultdict(int)
builds_summary['total'] = len(self.builds.all())
for build in self.builds:
state_name = ArtifactBuildState(build.state).name
builds_summary[state_name] += 1
data = self._common_json()
data['builds_summary'] = dict(builds_summary)
return data
def _common_json(self):
event_url = get_url_for('event', id=self.id)
db.session.add(self)
return {
"id":
self.id,
"message_id":
self.message_id,
"search_key":
self.search_key,
"event_type_id":
self.event_type_id,
"state":
self.state,
"state_name":
EventState(self.state).name,
"state_reason":
self.state_reason,
"time_created":
_utc_datetime_to_iso(self.time_created),
"time_done":
_utc_datetime_to_iso(self.time_done),
"url":
event_url,
"dry_run":
self.dry_run,
"requester":
self.requester,
"requested_rebuilds": (self.requested_rebuilds.split(" ")
if self.requested_rebuilds else []),
"requester_metadata":
self.requester_metadata_json,
"depends_on_events":
[event.id for event in self.event_dependencies],
"depending_events": [event.id for event in self.depending_events],
}
def find_dependent_events(self):
"""
Find other unreleased Events which built the same builds (or just some
of them) as this Event and adds them as a dependency for this event.
Dependent events of may also rebuild some same images that current event
will build. So, for building images found from current event, we also
need those YUM repositories used to build images in dependent events.
"""
builds_nvrs = [build.name for build in self.builds]
states = [
EventState.INITIALIZED.value, EventState.BUILDING.value,
EventState.COMPLETE.value
]
query = db.session.query(ArtifactBuild.event_id)
dep_event_ids = query.join(ArtifactBuild.event).filter(
ArtifactBuild.name.in_(builds_nvrs),
ArtifactBuild.event_id != self.id,
ArtifactBuild.type == ArtifactType.IMAGE.value,
Event.manual_triggered == false(),
Event.released == false(),
Event.state.in_(states),
).distinct()
dep_events = []
query = db.session.query(Event)
for row in dep_event_ids:
dep_event = query.filter_by(id=row[0]).first()
self.add_event_dependency(db.session, dep_event)
dep_events.append(dep_event)
db.session.commit()
return dep_events
def get_artifact_build_from_event_dependencies(self, nvr):
"""
It returns the artifact build, with `DONE` state, from the event dependencies (the build
of the parent event). `nvr` is used as `original_nvr` when finding the `ArtifactBuild`.
It returns all the parent artifact builds from the first found event dependency.
If the build is not found, it returns None.
"""
for parent_event in self.event_dependencies:
parent_build = db.session.query(ArtifactBuild).filter_by(
event_id=parent_event.id,
original_nvr=nvr,
state=ArtifactBuildState.DONE.value).all()
if parent_build:
return parent_build