class Repository(db.Model):
"""
Represents a VCS repository that Changes will watch for new commits.
"""
__tablename__ = 'repository'
id = Column(GUID, primary_key=True, default=uuid4)
url = Column(String(200), nullable=False, unique=True)
backend = Column(EnumType(RepositoryBackend),
default=RepositoryBackend.unknown,
nullable=False)
status = Column(EnumType(RepositoryStatus),
default=RepositoryStatus.inactive,
nullable=False,
server_default='1')
date_created = Column(DateTime, default=datetime.utcnow)
last_update = Column(DateTime)
last_update_attempt = Column(DateTime)
def __init__(self, **kwargs):
super(Repository, self).__init__(**kwargs)
if not self.id:
self.id = uuid4()
if not self.date_created:
self.date_created = datetime.utcnow()
def get_vcs(self):
from changes.models import ItemOption
from changes.vcs.git import GitVcs
from changes.vcs.hg import MercurialVcs
options = dict(
db.session.query(ItemOption.name, ItemOption.value).filter(
ItemOption.item_id == self.id,
ItemOption.name.in_([
'auth.username',
])))
kwargs = {
'path': os.path.join(current_app.config['REPO_ROOT'], self.id.hex),
'url': self.url,
'username': options.get('auth.username'),
}
if self.backend == RepositoryBackend.git:
return GitVcs(**kwargs)
elif self.backend == RepositoryBackend.hg:
return MercurialVcs(**kwargs)
else:
return None
@classmethod
def get(cls, id):
result = cls.query.filter_by(url=id).first()
if result is None and len(id) == 32:
result = cls.query.get(id)
return result
class Command(db.Model):
__tablename__ = 'command'
__table_args__ = (UniqueConstraint('jobstep_id',
'order',
name='unq_command_order'), )
id = Column(GUID, primary_key=True, default=uuid.uuid4)
jobstep_id = Column(GUID,
ForeignKey('jobstep.id', ondelete="CASCADE"),
nullable=False)
label = Column(String(128), nullable=False)
status = Column(EnumType(Status), nullable=False, default=Status.unknown)
return_code = Column(Integer, nullable=True)
script = Column(Text(), nullable=False)
env = Column(JSONEncodedDict, nullable=True)
cwd = Column(String(256), nullable=True)
artifacts = Column(ARRAY(String(256)), nullable=True)
date_started = Column(DateTime)
date_finished = Column(DateTime)
date_created = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
order = Column(Integer, default=0, server_default='0', nullable=False)
type = Column(EnumType(CommandType),
nullable=False,
default=CommandType.default,
server_default='0')
jobstep = relationship('JobStep',
backref=backref('commands',
order_by='Command.order'))
__repr__ = model_repr('jobstep_id', 'script')
def __init__(self, **kwargs):
super(Command, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.status is None:
self.status = Status.unknown
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.data is None:
self.data = {}
@property
def duration(self):
"""
Return the duration (in milliseconds) that this item was in-progress.
"""
if self.date_started and self.date_finished:
duration = (self.date_finished -
self.date_started).total_seconds() * 1000
else:
duration = None
return duration
class Plan(db.Model):
"""
Represents one of N build plans for a project.
"""
id = Column(GUID, primary_key=True, default=uuid4)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
label = Column(String(128), nullable=False)
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
date_modified = Column(DateTime, default=datetime.utcnow, nullable=False)
data = Column(JSONEncodedDict)
status = Column(EnumType(PlanStatus),
default=PlanStatus.inactive,
nullable=False,
server_default='1')
avg_build_time = Column(Integer)
project = relationship('Project', backref=backref('plans'))
__repr__ = model_repr('label')
__tablename__ = 'plan'
def __init__(self, **kwargs):
super(Plan, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
class Plan(db.Model):
"""
What work should we do for our new revision? A project may have multiple
plans, e.g. whenever a diff comes in, test it on both mac and windows
(each being its own plan.) In theory, a plan consists of a sequence of
steps; in practice, a plan is just a wrapper around a single step.
"""
id = Column(GUID, primary_key=True, default=uuid4)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
label = Column(String(128), nullable=False)
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
date_modified = Column(DateTime, default=datetime.utcnow, nullable=False)
data = Column(JSONEncodedDict)
status = Column(EnumType(PlanStatus),
default=PlanStatus.inactive,
nullable=False,
server_default='1')
# If not None, use snapshot from another plan. This allows us to share
# a single snapshot between multiple plans.
#
# This plan must be a plan from the same project (or else jobstep_details
# will fail) but this is not enforced by the database schema because we do
# not use a composite key.
snapshot_plan_id = Column(GUID,
ForeignKey('plan.id', ondelete="SET NULL"),
nullable=True)
avg_build_time = Column(Integer)
project = relationship('Project', backref=backref('plans'))
snapshot_plan = relationship('Plan', remote_side=[id])
__repr__ = model_repr('label')
__tablename__ = 'plan'
def __init__(self, **kwargs):
super(Plan, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
def get_item_options(self):
from changes.models.option import ItemOption
options_query = db.session.query(ItemOption.name,
ItemOption.value).filter(
ItemOption.item_id == self.id, )
options = dict()
for opt_name, opt_value in options_query:
options[opt_name] = opt_value
return options
def autogenerated(self):
return self.get_item_options().get('bazel.autogenerate', '0') == '1'
class TestArtifact(db.Model):
"""
Represents any artifacts generated by a single run of a single test. used
e.g. in server-selenium to store screenshots and large log dumps for
later debugging.
"""
__tablename__ = 'testartifact'
__tableargs__ = (
Index('idx_test_id', 'test_id'),
)
id = Column(GUID, nullable=False, primary_key=True, default=uuid.uuid4)
test_id = Column(GUID, ForeignKey('test.id', ondelete="CASCADE"), nullable=False)
name = Column('name', String(length=256), nullable=False)
type = Column(EnumType(TestArtifactType),
default=TestArtifactType.unknown,
nullable=False, server_default='0')
file = Column(FileStorage(**TESTARTIFACT_STORAGE_OPTIONS))
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
test = relationship('TestCase', backref='artifacts')
__repr__ = model_repr('name', 'type', 'file')
def __init__(self, **kwargs):
super(TestArtifact, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.date_created is None:
self.date_created = datetime.utcnow()
if isinstance(self.type, str):
self.type = TestArtifactType[self.type]
if self.file is None:
# TODO(dcramer): this is super hacky but not sure a better way to
# do it with SQLAlchemy
self.file = FileData({}, TESTARTIFACT_STORAGE_OPTIONS)
def save_base64_content(self, base64):
content = b64decode(base64)
self.file.save(
StringIO(content),
'{0}/{1}_{2}'.format(
self.test_id, self.id.hex, self.name
),
self._get_content_type()
)
def _get_content_type(self):
content_type, encoding = mimetypes.guess_type(self.name)
if content_type == 'text/html':
# upload html artifacts as plain text so the browser doesn't try to
# render them when viewing them raw
content_type = 'text/plain'
return content_type
class Snapshot(db.Model):
"""
Represents a snapshot used as a base in builds.
This is primarily used to indicate status and contains a collection of
SnapshotImage's.
"""
__tablename__ = 'snapshot'
id = Column(GUID, primary_key=True, default=uuid4)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
build_id = Column(GUID, ForeignKey('build.id'), unique=True)
source_id = Column(GUID, ForeignKey('source.id'))
status = Column(EnumType(SnapshotStatus),
default=SnapshotStatus.unknown,
nullable=False,
server_default='0')
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
build = relationship('Build')
project = relationship('Project', innerjoin=True)
source = relationship('Source')
def __init__(self, **kwargs):
super(Snapshot, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
@classmethod
def get_current(self, project_id):
from changes.models import ProjectOption
current_id = db.session.query(ProjectOption.value).filter(
ProjectOption.project_id == project_id,
ProjectOption.name == 'snapshot.current',
).scalar()
if not current_id:
return
return Snapshot.query.get(current_id)
class SnapshotImage(db.Model):
"""
Represents an individual image within a snapshot.
An image is bound to a (snapshot, plan) and represents the low level base
image that a snapshottable-job should be based on.
"""
__tablename__ = 'snapshot_image'
__table_args__ = (UniqueConstraint('snapshot_id',
'plan_id',
name='unq_snapshotimage_plan'), )
id = Column(GUID, primary_key=True, default=uuid4)
snapshot_id = Column(GUID,
ForeignKey('snapshot.id', ondelete="CASCADE"),
nullable=False)
plan_id = Column(GUID,
ForeignKey('plan.id', ondelete="CASCADE"),
nullable=False)
job_id = Column(GUID,
ForeignKey('job.id', ondelete="CASCADE"),
unique=True)
status = Column(EnumType(SnapshotStatus),
default=SnapshotStatus.unknown,
nullable=False,
server_default='0')
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
snapshot = relationship('Snapshot',
backref=backref(
'images',
order_by='SnapshotImage.date_created'))
plan = relationship('Plan')
job = relationship('Job')
def __init__(self, **kwargs):
super(SnapshotImage, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
class Snapshot(db.Model):
"""
A snapshot is a set of LXC container images (up to one for each plan in a project).
Each project can have an arbitrary number of snapshots, but only up
to one "current snapshot" is actually used by builds (stored as
ProjectOption) at any time.
Snapshots are used in the Mesos and Jenkins-LXC environments.
Snapshots are currently only used with changes-client.
When running a build, the images of the current snapshot are used
for individual jobs that are part of a build. A snapshot image
can be shared between multiple plans by setting snapshot_plan_id
of a Plan. By default, there is a separate image for each plan
of a build.
The status of a snapshot indicates whether it *can* be used for
builds; it doesn't indicate whether the snapshot is actually used
for builds right now. A snapshot is active if and only if all the
corresponding snapshot images are active.
A snapshot is generated by a slightly special snapshot build that
uploads a snapshot at the end of the build.
"""
__tablename__ = 'snapshot'
id = Column(GUID, primary_key=True, default=uuid4)
project_id = Column(
GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False)
build_id = Column(GUID, ForeignKey('build.id'), unique=True)
source_id = Column(GUID, ForeignKey('source.id'))
# Most importantly, this tells if this snapshot can be used for builds (i.e., are all
# component snapshot images active)?
status = Column(EnumType(SnapshotStatus),
default=SnapshotStatus.unknown,
nullable=False, server_default='0')
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
# The build that generated this snapshot.
build = relationship('Build', backref=backref('snapshot', uselist=False))
project = relationship('Project', innerjoin=True)
# The source that was used to generate the snapshot.
source = relationship('Source')
def __init__(self, **kwargs):
super(Snapshot, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
@classmethod
def get_current(cls, project_id):
"""Return the current Snapshot for a project (or None if one is not set)."""
from changes.models import ProjectOption
current_id = db.session.query(ProjectOption.value).filter(
ProjectOption.project_id == project_id,
ProjectOption.name == 'snapshot.current',
).scalar()
if not current_id:
return
return Snapshot.query.get(current_id)
class SnapshotImage(db.Model):
"""
Represents an individual image within a snapshot.
An image is bound to a (snapshot, plan) and represents the low level base
image that a snapshottable-job should be based on.
Note that a project with multiple plans may have multiple, independent
images per snapshot, as images aren't always shared between plans.
"""
__tablename__ = 'snapshot_image'
__table_args__ = (
UniqueConstraint('snapshot_id', 'plan_id', name='unq_snapshotimage_plan'),
)
# The snapshot image id is used by changes-client to store and retrieve snapshots.
# New ids are created by changes and passed on to changes-client.
id = Column(GUID, primary_key=True, default=uuid4)
snapshot_id = Column(
GUID, ForeignKey('snapshot.id', ondelete="CASCADE"), nullable=False)
plan_id = Column(
GUID, ForeignKey('plan.id', ondelete="CASCADE"), nullable=False)
job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), unique=True)
status = Column(EnumType(SnapshotStatus),
default=SnapshotStatus.unknown,
nullable=False, server_default='0')
date_created = Column(DateTime, default=datetime.utcnow, nullable=False)
snapshot = relationship('Snapshot', backref=backref('images',
order_by='SnapshotImage.date_created'))
plan = relationship('Plan')
# The job that was used to create this snapshot.
job = relationship('Job')
def __init__(self, **kwargs):
super(SnapshotImage, self).__init__(**kwargs)
if self.id is None:
self.id = uuid4()
def change_status(self, status):
"""
The status field of snapshot is a redundant field that has to
be maintained. Its essentially a cached aggregate over
snapshot_image.status. This means that whenever we update
snapshot_image.status we have to update snapshot.status.
This method updates the current status to the new status given
as a parameter.
TODO we should probably verify that if the current status is
active and we are tring to move it to a new status that is
not "invalidated" we should give some error.
XXX(jhance)
Is this a sign of a defective schema? Computing snapshot.status
should be possible in-query although I'm not sure to do it from
within sqlalchemy.
"""
self.status = status
db.session.add(self)
# We need to update the current database with the status of the
# new image, but we don't commit completely until we have found
# the status of the overall status and update it atomically
db.session.flush()
inactive_image_query = SnapshotImage.query.filter(
SnapshotImage.status != SnapshotStatus.active,
SnapshotImage.snapshot_id == self.snapshot.id,
).exists()
if not db.session.query(inactive_image_query).scalar():
# If the snapshot status isn't pending for whatever reason, then we
# refuse to update its status to active because clearly some other
# error occurred elsewhere in the pipeline (for example, the
# snapshot build itself failing)
if self.snapshot.status == SnapshotStatus.pending:
self.snapshot.status = SnapshotStatus.active
elif self.snapshot.status == SnapshotStatus.active:
self.snapshot.status = SnapshotStatus.invalidated
db.session.commit()
@classmethod
def get(cls, plan, snapshot_id):
"""Return the SnapshotImage for a plan or None if one is not set.
"""
# This plan might be configured to be dependent on another plan's snapshot
snapshot_plan = plan
if plan.snapshot_plan is not None:
snapshot_plan = plan.snapshot_plan
snapshot = Snapshot.query.filter(Snapshot.id == snapshot_id).scalar()
if snapshot is not None:
return SnapshotImage.query.filter(
SnapshotImage.snapshot_id == snapshot.id,
SnapshotImage.plan_id == snapshot_plan.id,
).scalar()
return None
class Build(db.Model):
"""
Represents the work we do (e.g. running tests) for one diff or commit (an
entry in the source table) in one particular project
Each Build contains many Jobs (usually linked to a JobPlan).
"""
__tablename__ = 'build'
__table_args__ = (
Index('idx_buildfamily_project_id', 'project_id'),
Index('idx_buildfamily_author_id', 'author_id'),
Index('idx_buildfamily_source_id', 'source_id'),
UniqueConstraint('project_id', 'number', name='unq_build_number'),
)
id = Column(GUID, primary_key=True, default=uuid.uuid4)
number = Column(Integer)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
# A unqiue identifier for a group of related Builds, such as all Builds created by a particular
# action. Used primarily for aggregation in result reporting.
# Note that this may be None for Builds that aren't grouped, and all such Builds should NOT
# be treated as a collection.
collection_id = Column(GUID)
source_id = Column(GUID, ForeignKey('source.id', ondelete="CASCADE"))
author_id = Column(GUID, ForeignKey('author.id', ondelete="CASCADE"))
cause = Column(EnumType(Cause), nullable=False, default=Cause.unknown)
# label is a short description, typically from the title of the change that triggered the build.
label = Column(String(128), nullable=False)
# short indicator of what is being built, typically the sha or the Phabricator revision ID like 'D90885'.
target = Column(String(128))
tags = Column(ARRAY(String(16)), nullable=True)
status = Column(EnumType(Status), nullable=False, default=Status.unknown)
result = Column(EnumType(Result), nullable=False, default=Result.unknown)
message = Column(Text)
duration = Column(Integer)
priority = Column(EnumType(BuildPriority),
nullable=False,
default=BuildPriority.default,
server_default='0')
date_started = Column(DateTime)
date_finished = Column(DateTime)
date_created = Column(DateTime, default=datetime.utcnow)
date_modified = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
project = relationship('Project', innerjoin=True)
source = relationship('Source', innerjoin=True)
author = relationship('Author')
stats = relationship('ItemStat',
primaryjoin='Build.id == ItemStat.item_id',
foreign_keys=[id],
uselist=True)
__repr__ = model_repr('label', 'target')
def __init__(self, **kwargs):
super(Build, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.result is None:
self.result = Result.unknown
if self.status is None:
self.status = Status.unknown
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
if self.date_started and self.date_finished and not self.duration:
self.duration = (self.date_finished -
self.date_started).total_seconds() * 1000
if self.number is None and self.project:
self.number = select([func.next_item_value(self.project.id.hex)])
class Command(db.Model):
"""
The information of the script run on one node within a jobstep: the contents
of the script are included, and later the command can be updated
with status/return code.
changes-client has no real magic beyond running commands, so the list
of commands it ran basically tells you everything that happened.
Looks like only mesos/lxc builds (DefaultBuildStep)
"""
__tablename__ = 'command'
__table_args__ = (UniqueConstraint('jobstep_id',
'order',
name='unq_command_order'), )
id = Column(GUID, primary_key=True, default=uuid.uuid4)
jobstep_id = Column(GUID,
ForeignKey('jobstep.id', ondelete="CASCADE"),
nullable=False)
label = Column(String(128), nullable=False)
status = Column(EnumType(Status), nullable=False, default=Status.unknown)
return_code = Column(Integer, nullable=True)
script = Column(Text(), nullable=False)
env = Column(JSONEncodedDict, nullable=True)
cwd = Column(String(256), nullable=True)
artifacts = Column(ARRAY(String(256)), nullable=True)
date_started = Column(DateTime)
date_finished = Column(DateTime)
date_created = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
order = Column(Integer, default=0, server_default='0', nullable=False)
type = Column(EnumType(CommandType),
nullable=False,
default=CommandType.default,
server_default='0')
jobstep = relationship('JobStep',
backref=backref('commands',
order_by='Command.order'))
__repr__ = model_repr('jobstep_id', 'script')
def __init__(self, **kwargs):
super(Command, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.status is None:
self.status = Status.unknown
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.data is None:
self.data = {}
@property
def duration(self):
"""
Return the duration (in milliseconds) that this item was in-progress.
"""
if self.date_started and self.date_finished:
duration = (self.date_finished -
self.date_started).total_seconds() * 1000
else:
duration = None
return duration
class Build(db.Model):
"""
Represents a collection of builds for a single target, as well as the sum
of their results.
Each Build contains many Jobs (usually linked to a JobPlan).
"""
__tablename__ = 'build'
__table_args__ = (
Index('idx_buildfamily_project_id', 'project_id'),
Index('idx_buildfamily_author_id', 'author_id'),
Index('idx_buildfamily_source_id', 'source_id'),
UniqueConstraint('project_id', 'number', name='unq_build_number'),
)
id = Column(GUID, primary_key=True, default=uuid.uuid4)
number = Column(Integer)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
collection_id = Column(GUID)
source_id = Column(GUID, ForeignKey('source.id', ondelete="CASCADE"))
author_id = Column(GUID, ForeignKey('author.id', ondelete="CASCADE"))
cause = Column(EnumType(Cause), nullable=False, default=Cause.unknown)
label = Column(String(128), nullable=False)
target = Column(String(128))
tags = Column(ARRAY(String(16)), nullable=True)
status = Column(EnumType(Status), nullable=False, default=Status.unknown)
result = Column(EnumType(Result), nullable=False, default=Result.unknown)
message = Column(Text)
duration = Column(Integer)
priority = Column(EnumType(BuildPriority),
nullable=False,
default=BuildPriority.default,
server_default='0')
date_started = Column(DateTime)
date_finished = Column(DateTime)
date_created = Column(DateTime, default=datetime.utcnow)
date_modified = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
project = relationship('Project', innerjoin=True)
source = relationship('Source', innerjoin=True)
author = relationship('Author')
stats = relationship('ItemStat',
primaryjoin='Build.id == ItemStat.item_id',
foreign_keys=[id],
uselist=True)
__repr__ = model_repr('label', 'target')
def __init__(self, **kwargs):
super(Build, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.result is None:
self.result = Result.unknown
if self.status is None:
self.status = Status.unknown
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
if self.date_started and self.date_finished and not self.duration:
self.duration = (self.date_finished -
self.date_started).total_seconds() * 1000
if self.number is None and self.project:
self.number = select([func.next_item_value(self.project.id.hex)])