class Plan(db.Model): """ Represents one of N build plans for a project. """ id = Column(GUID, primary_key=True, default=uuid4) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) date_modified = Column(DateTime, default=datetime.utcnow, nullable=False) data = Column(JSONEncodedDict) status = Column(EnumType(PlanStatus), default=PlanStatus.inactive, nullable=False, server_default='1') avg_build_time = Column(Integer) project = relationship('Project', backref=backref('plans')) __repr__ = model_repr('label') __tablename__ = 'plan' def __init__(self, **kwargs): super(Plan, self).__init__(**kwargs) if self.id is None: self.id = uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class Node(db.Model): """ A machine that runs jobsteps. This is populated by observing the machines picked by the jenkins masters (which themselves are configured by BuildStep params in the changes UI) when they're asked to run task, and is not configured manually. Node machines have tags (not stored in the changes db) """ __tablename__ = 'node' id = Column(GUID, primary_key=True, default=uuid.uuid4) label = Column(String(128), unique=True) data = Column(JSONEncodedDict) date_created = Column(DateTime, default=datetime.utcnow) clusters = association_proxy('node_clusters', 'cluster') __repr__ = model_repr('label') def __init__(self, *args, **kwargs): super(Node, self).__init__(*args, **kwargs) if not self.id: self.id = uuid.uuid4()
class Step(db.Model): """ Represents one of N build steps for a plan. """ # TODO(dcramer): only a single step is currently supported id = Column(GUID, primary_key=True, default=uuid4) plan_id = Column(GUID, ForeignKey('plan.id', ondelete='CASCADE'), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) date_modified = Column(DateTime, default=datetime.utcnow, nullable=False) # implementation should be class path notation implementation = Column(String(128), nullable=False) order = Column(Integer, nullable=False) data = Column(JSONEncodedDict) plan = relationship('Plan', backref=backref('steps', order_by='Step.order')) __repr__ = model_repr('plan_id', 'implementation') __tablename__ = 'step' __table_args__ = ( UniqueConstraint('plan_id', 'order', name='unq_plan_key'), CheckConstraint(order >= 0, name='chk_step_order_positive'), ) def __init__(self, **kwargs): super(Step, self).__init__(**kwargs) if self.id is None: self.id = uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created def get_implementation(self): return import_string(self.implementation)(**self.data)
class BuildSeen(db.Model): """ Keeps track of when users have viewed builds in the ui. Not sure we expose this to users in the ui right now. """ __tablename__ = 'buildseen' __table_args__ = (UniqueConstraint('build_id', 'user_id', name='unq_buildseen_entity'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) build_id = Column(GUID, ForeignKey('build.id', ondelete="CASCADE"), nullable=False) user_id = Column(GUID, ForeignKey('user.id', ondelete="CASCADE"), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) build = relationship('Build') user = relationship('User') __repr__ = model_repr('build_id', 'user_id') def __init__(self, **kwargs): super(BuildSeen, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow()
class Event(db.Model): __tablename__ = 'event' __table_args__ = ( Index('idx_event_item_id', 'item_id'), # Having this as unique prevents duplicate events, but in the future # we may want to allow duplicates # e.g. we can have a "sent email notification" event, but maybe # we'd want to have multiple of those UniqueConstraint('type', 'item_id', name='unq_event_key'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) type = Column(String(32), nullable=False) item_id = Column('item_id', GUID, nullable=False) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) __repr__ = model_repr('type', 'item_id') def __init__(self, **kwargs): super(Event, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class Job(db.Model): __tablename__ = 'job' __table_args__ = ( Index('idx_build_project_id', 'project_id'), Index('idx_build_change_id', 'change_id'), Index('idx_build_source_id', 'source_id'), Index('idx_build_family_id', 'build_id'), UniqueConstraint('build_id', 'number', name='unq_job_number'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) number = Column(Integer) # TODO(dcramer): change should be removed in favor of an m2m between # Change and Source build_id = Column(GUID, ForeignKey('build.id', ondelete="CASCADE")) change_id = Column(GUID, ForeignKey('change.id', ondelete="CASCADE")) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) source_id = Column(GUID, ForeignKey('source.id', ondelete="CASCADE")) label = Column(String(128), nullable=False) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) duration = Column(Integer) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) change = relationship('Change') build = relationship('Build', backref=backref('jobs', order_by='Job.number'), innerjoin=True) project = relationship('Project') source = relationship('Source') __repr__ = model_repr('label', 'target') def __init__(self, **kwargs): super(Job, self).__init__(**kwargs) if self.data is None: self.data = {} if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created if self.date_started and self.date_finished and not self.duration: self.duration = (self.date_finished - self.date_started).total_seconds() * 1000 if self.number is None and self.build: self.number = select([func.next_item_value(self.build.id.hex)])
class JobStep(db.Model): """ The most granular unit of work; run on a particular node, has a status and a result. """ __tablename__ = 'jobstep' __table_args__ = ( Index('idx_jobstep_status', 'status'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) phase_id = Column(GUID, ForeignKey('jobphase.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) node_id = Column(GUID, ForeignKey('node.id', ondelete="CASCADE")) # id of JobStep that replaces this JobStep. Usually None, unless a JobStep # fails and is retried. replacement_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"), unique=True) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) last_heartbeat = Column(DateTime) data = Column(JSONEncodedDict) job = relationship('Job') project = relationship('Project') node = relationship('Node') phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started')) __repr__ = model_repr('label') def __init__(self, **kwargs): super(JobStep, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class Plan(db.Model): """ What work should we do for our new revision? A project may have multiple plans, e.g. whenever a diff comes in, test it on both mac and windows (each being its own plan.) In theory, a plan consists of a sequence of steps; in practice, a plan is just a wrapper around a single step. """ id = Column(GUID, primary_key=True, default=uuid4) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) date_modified = Column(DateTime, default=datetime.utcnow, nullable=False) data = Column(JSONEncodedDict) status = Column(EnumType(PlanStatus), default=PlanStatus.inactive, nullable=False, server_default='1') # If not None, use snapshot from another plan. This allows us to share # a single snapshot between multiple plans. # # This plan must be a plan from the same project (or else jobstep_details # will fail) but this is not enforced by the database schema because we do # not use a composite key. snapshot_plan_id = Column(GUID, ForeignKey('plan.id', ondelete="SET NULL"), nullable=True) avg_build_time = Column(Integer) project = relationship('Project', backref=backref('plans')) snapshot_plan = relationship('Plan', remote_side=[id]) __repr__ = model_repr('label') __tablename__ = 'plan' def __init__(self, **kwargs): super(Plan, self).__init__(**kwargs) if self.id is None: self.id = uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created def get_item_options(self): from changes.models.option import ItemOption options_query = db.session.query(ItemOption.name, ItemOption.value).filter( ItemOption.item_id == self.id, ) options = dict() for opt_name, opt_value in options_query: options[opt_name] = opt_value return options def autogenerated(self): return self.get_item_options().get('bazel.autogenerate', '0') == '1'
class Cluster(db.Model): __tablename__ = 'cluster' id = Column(GUID, primary_key=True, default=uuid.uuid4) label = Column(String(128), unique=True) date_created = Column(DateTime, default=datetime.utcnow) plans = association_proxy('cluster_nodes', 'node') __repr__ = model_repr('label')
class Command(db.Model): __tablename__ = 'command' __table_args__ = (UniqueConstraint('jobstep_id', 'order', name='unq_command_order'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) jobstep_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(EnumType(Status), nullable=False, default=Status.unknown) return_code = Column(Integer, nullable=True) script = Column(Text(), nullable=False) env = Column(JSONEncodedDict, nullable=True) cwd = Column(String(256), nullable=True) artifacts = Column(ARRAY(String(256)), nullable=True) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) order = Column(Integer, default=0, server_default='0', nullable=False) type = Column(EnumType(CommandType), nullable=False, default=CommandType.default, server_default='0') jobstep = relationship('JobStep', backref=backref('commands', order_by='Command.order')) __repr__ = model_repr('jobstep_id', 'script') def __init__(self, **kwargs): super(Command, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class Step(db.Model): """ A specific description of how to do some work for a build. In theory, a plan can have multiple steps. In practice, every plan has only one step and plan is just a thin wrapper around step. Steps are not freeform, rather, each step is just configuration data for specific step implementations that are hard-coded in python. """ # TODO(dcramer): only a single step is currently supported id = Column(GUID, primary_key=True, default=uuid4) plan_id = Column(GUID, ForeignKey('plan.id', ondelete='CASCADE'), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) date_modified = Column(DateTime, default=datetime.utcnow, nullable=False) # implementation should be class path notation implementation = Column(String(128), nullable=False) order = Column(Integer, nullable=False) data = Column(JSONEncodedDict) plan = relationship('Plan', backref=backref('steps', order_by='Step.order')) __repr__ = model_repr('plan_id', 'implementation') __tablename__ = 'step' __table_args__ = ( UniqueConstraint('plan_id', 'order', name='unq_plan_key'), CheckConstraint(order >= 0, name='chk_step_order_positive'), ) def __init__(self, **kwargs): super(Step, self).__init__(**kwargs) if self.id is None: self.id = uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created def get_implementation(self, load=True): try: cls = import_string(self.implementation) except Exception: return None if not load: return cls try: # XXX(dcramer): It's important that we deepcopy data so any # mutations within the BuildStep don't propagate into the db return cls(**deepcopy(self.data)) except Exception: return None
class TestArtifact(db.Model): """ Represents any artifacts generated by a single run of a single test. used e.g. in server-selenium to store screenshots and large log dumps for later debugging. """ __tablename__ = 'testartifact' __tableargs__ = ( Index('idx_test_id', 'test_id'), ) id = Column(GUID, nullable=False, primary_key=True, default=uuid.uuid4) test_id = Column(GUID, ForeignKey('test.id', ondelete="CASCADE"), nullable=False) name = Column('name', String(length=256), nullable=False) type = Column(EnumType(TestArtifactType), default=TestArtifactType.unknown, nullable=False, server_default='0') file = Column(FileStorage(**TESTARTIFACT_STORAGE_OPTIONS)) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) test = relationship('TestCase', backref='artifacts') __repr__ = model_repr('name', 'type', 'file') def __init__(self, **kwargs): super(TestArtifact, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if isinstance(self.type, str): self.type = TestArtifactType[self.type] if self.file is None: # TODO(dcramer): this is super hacky but not sure a better way to # do it with SQLAlchemy self.file = FileData({}, TESTARTIFACT_STORAGE_OPTIONS) def save_base64_content(self, base64): content = b64decode(base64) self.file.save( StringIO(content), '{0}/{1}_{2}'.format( self.test_id, self.id.hex, self.name ), self._get_content_type() ) def _get_content_type(self): content_type, encoding = mimetypes.guess_type(self.name) if content_type == 'text/html': # upload html artifacts as plain text so the browser doesn't try to # render them when viewing them raw content_type = 'text/plain' return content_type
class Task(db.Model): __tablename__ = 'task' __table_args__ = ( Index('idx_task_parent_id', 'parent_id', 'task_name'), Index('idx_task_child_id', 'child_id', 'task_name'), UniqueConstraint('task_name', 'parent_id', 'child_id', name='unq_task_entity'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) task_name = Column(String(128), nullable=False) task_id = Column('child_id', GUID, nullable=False) parent_id = Column(GUID) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) num_retries = Column(Integer, nullable=False, default=0) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) __repr__ = model_repr('task_name', 'parent_id', 'child_id', 'status') def __init__(self, **kwargs): super(Task, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created @classmethod def check(cls, task_name, parent_id): """ >>> if Task.check('my_task', parent_item.id) == Status.finished: >>> print "all child tasks done!" """ # XXX(dcramer): we could make this fast if we're concerneda bout # of # rows by doing two network hops (first check for in progress, then # report result) child_tasks = list( db.session.query(cls.result, Task.status).filter( cls.task_name == task_name, cls.parent_id == parent_id, )) if any(r.status != Status.finished for r in child_tasks): return Status.in_progress return Status.finished
class JobStep(db.Model): # TODO(dcramer): make duration a column __tablename__ = 'jobstep' id = Column(GUID, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) phase_id = Column(GUID, ForeignKey('jobphase.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) node_id = Column(GUID, ForeignKey('node.id', ondelete="CASCADE")) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) last_heartbeat = Column(DateTime) data = Column(JSONEncodedDict) job = relationship('Job') project = relationship('Project') node = relationship('Node') phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started')) __repr__ = model_repr('label') def __init__(self, **kwargs): super(JobStep, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Result.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class Node(db.Model): __tablename__ = 'node' id = Column(GUID, primary_key=True, default=uuid.uuid4) label = Column(String(128), unique=True) data = Column(JSONEncodedDict) date_created = Column(DateTime, default=datetime.utcnow) clusters = association_proxy('node_clusters', 'cluster') __repr__ = model_repr('label') def __init__(self, *args, **kwargs): super(Node, self).__init__(*args, **kwargs) if not self.id: self.id = uuid.uuid4()
class ItemStat(db.Model): __tablename__ = 'itemstat' __table_args__ = (UniqueConstraint('item_id', 'name', name='unq_itemstat_name'), ) id = Column(GUID, primary_key=True, default=uuid4) item_id = Column(GUID, nullable=False) name = Column(String(64), nullable=False) value = Column(Integer, nullable=False) __repr__ = model_repr('item_id', 'name', 'value') def __init__(self, **kwargs): super(ItemStat, self).__init__(**kwargs) if self.id is None: self.id = uuid4()
class JobPlan(db.Model): """ A link to all Job + Plan's for a Build. TODO(dcramer): this should include a snapshot of the plan at build time. """ __tablename__ = 'jobplan' __table_args__ = ( Index('idx_buildplan_project_id', 'project_id'), Index('idx_buildplan_family_id', 'build_id'), Index('idx_buildplan_plan_id', 'plan_id'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) build_id = Column(GUID, ForeignKey('build.id', ondelete="CASCADE"), nullable=False) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False, unique=True) plan_id = Column(GUID, ForeignKey('plan.id', ondelete="CASCADE"), nullable=False) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) project = relationship('Project') build = relationship('Build') job = relationship('Job') plan = relationship('Plan') __repr__ = model_repr('build_id', 'job_id', 'plan_id') def __init__(self, **kwargs): super(JobPlan, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class Cluster(db.Model): """ A group of nodes. We refer to clusters in the step configurations (where should we run our tests?) Clusters are automatically added when we see them from jenkins results. Apparently, clusters are only used in jenkins (not lxc, although nodes are used for both.) A cluster does not correspond to one master """ __tablename__ = 'cluster' id = Column(GUID, primary_key=True, default=uuid.uuid4) label = Column(String(128), unique=True) date_created = Column(DateTime, default=datetime.utcnow) plans = association_proxy('cluster_nodes', 'node') __repr__ = model_repr('label')
class ItemStat(db.Model): """ Also a key/value table, tailored towards statistics generated by tests and code coverage. Examples: test_rerun_count, test_duration, lines_covered """ __tablename__ = 'itemstat' __table_args__ = (UniqueConstraint('item_id', 'name', name='unq_itemstat_name'), ) id = Column(GUID, primary_key=True, default=uuid4) item_id = Column(GUID, nullable=False) name = Column(String(64), nullable=False) value = Column(Integer, nullable=False) __repr__ = model_repr('item_id', 'name', 'value') def __init__(self, **kwargs): super(ItemStat, self).__init__(**kwargs) if self.id is None: self.id = uuid4()
class TestSuite(db.Model): """ A test suite is usually representive of the tooling running the tests. Tests are unique per test suite. """ __tablename__ = 'testsuite' __table_args__ = ( UniqueConstraint('job_id', 'name_sha', name='_suite_key'), Index('idx_testsuite_project_id', 'project_id'), ) id = Column(GUID, nullable=False, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) name_sha = Column(String(40), nullable=False, default=sha1('default').hexdigest()) name = Column(Text, nullable=False, default='default') date_created = Column(DateTime, default=datetime.utcnow, nullable=False) job = relationship('Job') project = relationship('Project') __repr__ = model_repr('name') def __init__(self, **kwargs): super(TestSuite, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.name is None: self.name = 'default'
class Plan(db.Model): """ Represents one of N build plans for a project. """ id = Column(GUID, primary_key=True, default=uuid4) label = Column(String(128), nullable=False) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) date_modified = Column(DateTime, default=datetime.utcnow, nullable=False) data = Column(JSONEncodedDict) projects = association_proxy('plan_projects', 'project') __repr__ = model_repr('label') __tablename__ = 'plan' def __init__(self, **kwargs): super(Plan, self).__init__(**kwargs) if self.id is None: self.id = uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class Event(db.Model): """ Indicates that something (specified by `type` and `data`) happened to some entity (specified by `item_id`). This allows us to record that we've performed some action with an external side-effect so that we can be sure we do it no more than once. It is also useful for displaying to users which actions have been performed when, and whether they were successful. """ __tablename__ = 'event' __table_args__ = ( Index('idx_event_item_id', 'item_id'), # Having this as unique prevents duplicate events, but in the future # we may want to allow duplicates # e.g. we can have a "sent email notification" event, but maybe # we'd want to have multiple of those UniqueConstraint('type', 'item_id', name='unq_event_key'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) # A value from EventType type = Column(String(32), nullable=False) item_id = Column('item_id', GUID, nullable=False) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) __repr__ = model_repr('type', 'item_id') def __init__(self, **kwargs): super(Event, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class Event(db.Model): """ "No component of the system depends on event existing" its just for logging and displaying to the user. We log whenever we email a user about a broken build (or a green build, if that option is set in the ui.) Technically, the type column only has two distinct values: [email_notification, green_build_notification]. Contains a JSON data-blob """ __tablename__ = 'event' __table_args__ = ( Index('idx_event_item_id', 'item_id'), # Having this as unique prevents duplicate events, but in the future # we may want to allow duplicates # e.g. we can have a "sent email notification" event, but maybe # we'd want to have multiple of those UniqueConstraint('type', 'item_id', name='unq_event_key'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) type = Column(String(32), nullable=False) item_id = Column('item_id', GUID, nullable=False) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) __repr__ = model_repr('type', 'item_id') def __init__(self, **kwargs): super(Event, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created
class JobStep(db.Model): """ The most granular unit of work; run on a particular node, has a status and a result. """ __tablename__ = 'jobstep' __table_args__ = ( Index('idx_jobstep_status', 'status'), Index('idx_jobstep_cluster', 'cluster'), Index('idx_jobstep_project_date', 'project_id', 'date_created'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) phase_id = Column(GUID, ForeignKey('jobphase.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) node_id = Column(GUID, ForeignKey('node.id', ondelete="CASCADE")) # id of JobStep that replaces this JobStep. Usually None, unless a JobStep # fails and is retried. replacement_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"), unique=True) # Used (for non-Jenkins builds) in jobstep_allocate to only allocate jobsteps # to slaves of a particular cluster. For Jenkins builds, this is pure documentation (typically # set to the Jenkins label), but should be accurate just the same. cluster = Column(String(128), nullable=True) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) # The time of the last external interaction indicating progress. last_heartbeat = Column(DateTime) data = Column(JSONEncodedDict) job = relationship('Job') project = relationship('Project') node = relationship('Node') phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started')) targets = relationship(BazelTarget, backref=backref('step')) __repr__ = model_repr('label') def __init__(self, **kwargs): super(JobStep, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class JobStep(db.Model): """ The most granular unit of work; run on a particular node, has a status and a result. But Hark! There's a hack that allows jobstep, once its run, to rewrite history to say that it was actually multiple jobsteps (even organized into separate job phases.) It does this by creating an artifact, which the python code picks up and then retroactively alters the db to say that this jobstep had multiple steps (I think it purely appends new jobsteps after the original.) xplat uses this to very nicely display the different parts of their jobstep. """ # TODO(dcramer): make duration a column __tablename__ = 'jobstep' __table_args__ = ( Index('idx_jobstep_status', 'status'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) phase_id = Column(GUID, ForeignKey('jobphase.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) node_id = Column(GUID, ForeignKey('node.id', ondelete="CASCADE")) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) last_heartbeat = Column(DateTime) data = Column(JSONEncodedDict) job = relationship('Job') project = relationship('Project') node = relationship('Node') phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started')) __repr__ = model_repr('label') def __init__(self, **kwargs): super(JobStep, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class Task(db.Model): """ When we enqueue a task, we also write a db row to keep track of the task's metadata (e.g. number of times retried.) There is a slightly icky custom data column that each task type uses in its own way. This db represents serialized version of tracked_task you see in the changes python codebase. Tasks can have parent tasks. Parent tasks have the option of waiting for their children to complete (in practice, that always happens.) Example: sync_job with sync_jobstep children Tasks can throw a NotFinished exception, which will just mean that we try running it again after some interval (but this has nothing to do with retrying tasks that error!) Examples: Tasks with children will check to see if their children are finished; the sync_jobstep task will query jenkins to see if its finished. Tasks can fire signals, e.g. build xxx has finished. There's a table that maps signal types to tasks that should be created. Signals/listeners are not tracked as children of other tasks. """ __tablename__ = 'task' __table_args__ = ( Index('idx_task_parent_id', 'parent_id', 'task_name'), Index('idx_task_child_id', 'child_id', 'task_name'), Index('idx_task_date_created', 'date_created'), UniqueConstraint('task_name', 'parent_id', 'child_id', name='unq_task_entity'), Index('idx_task_status', 'status'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) task_name = Column(String(128), nullable=False) # TODO: Rename 'task_id' to 'child_id' in code to make things less confusing. task_id = Column('child_id', GUID, nullable=False) parent_id = Column(GUID) status = Column(Enum(Status), nullable=False, default=Status.unknown) result = Column(Enum(Result), nullable=False, default=Result.unknown) num_retries = Column(Integer, nullable=False, default=0) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) __repr__ = model_repr('task_name', 'parent_id', 'child_id', 'status') def __init__(self, **kwargs): super(Task, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created @classmethod def check(cls, task_name, parent_id): """ >>> if Task.check('my_task', parent_item.id) == Status.finished: >>> print "all child tasks done!" """ # XXX(dcramer): we could make this fast if we're concerned about # of # rows by doing two network hops (first check for in progress, then # report result) child_tasks = list( db.session.query(cls.result, Task.status).filter( cls.task_name == task_name, cls.parent_id == parent_id, )) if any(r.status != Status.finished for r in child_tasks): return Status.in_progress return Status.finished
class TestCase(db.Model): """ An individual test result. """ __tablename__ = 'test' __table_args__ = ( UniqueConstraint('job_id', 'label_sha', name='unq_test_name'), Index('idx_test_step_id', 'step_id'), Index('idx_test_project_key', 'project_id', 'label_sha'), ) id = Column(GUID, nullable=False, primary_key=True, default=uuid.uuid4) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) step_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE")) name_sha = Column('label_sha', String(40), nullable=False) name = Column(Text, nullable=False) _package = Column('package', Text, nullable=True) result = Column(Enum(Result), default=Result.unknown, nullable=False) duration = Column(Integer, default=0) message = deferred(Column(Text)) date_created = Column(DateTime, default=datetime.utcnow, nullable=False) reruns = Column(Integer) job = relationship('Job') step = relationship('JobStep') project = relationship('Project') __repr__ = model_repr('name', '_package', 'result') def __init__(self, **kwargs): super(TestCase, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.date_created is None: self.date_created = datetime.utcnow() @classmethod def calculate_name_sha(self, name): if name: return sha1(name).hexdigest() raise ValueError @property def sep(self): name = (self._package or self.name) # handle the case where it might begin with some special character if not re.match(r'^[a-zA-Z0-9]', name): return '/' elif '/' in name: return '/' return '.' def _get_package(self): if not self._package: try: package, _ = self.name.rsplit(self.sep, 1) except ValueError: package = None else: package = self._package return package def _set_package(self, value): self._package = value package = property(_get_package, _set_package) @property def short_name(self): name, package = self.name, self.package if package and name.startswith(package) and name != package: return name[len(package) + 1:] return name
class Command(db.Model): """ The information of the script run on one node within a jobstep: the contents of the script are included, and later the command can be updated with status/return code. changes-client has no real magic beyond running commands, so the list of commands it ran basically tells you everything that happened. Looks like only mesos/lxc builds (DefaultBuildStep) """ __tablename__ = 'command' __table_args__ = (UniqueConstraint('jobstep_id', 'order', name='unq_command_order'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) jobstep_id = Column(GUID, ForeignKey('jobstep.id', ondelete="CASCADE"), nullable=False) label = Column(String(128), nullable=False) status = Column(EnumType(Status), nullable=False, default=Status.unknown) return_code = Column(Integer, nullable=True) script = Column(Text(), nullable=False) env = Column(JSONEncodedDict, nullable=True) cwd = Column(String(256), nullable=True) artifacts = Column(ARRAY(String(256)), nullable=True) date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) order = Column(Integer, default=0, server_default='0', nullable=False) type = Column(EnumType(CommandType), nullable=False, default=CommandType.default, server_default='0') jobstep = relationship('JobStep', backref=backref('commands', order_by='Command.order')) __repr__ = model_repr('jobstep_id', 'script') def __init__(self, **kwargs): super(Command, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.data is None: self.data = {} @property def duration(self): """ Return the duration (in milliseconds) that this item was in-progress. """ if self.date_started and self.date_finished: duration = (self.date_finished - self.date_started).total_seconds() * 1000 else: duration = None return duration
class JobPlan(db.Model): """ A snapshot of a plan and its constituent steps, taken at job creation time. This exists so that running jobs are not impacted by configuration changes. Note that this table combines the data from the plan and step tables. """ __tablename__ = 'jobplan' __table_args__ = ( Index('idx_buildplan_project_id', 'project_id'), Index('idx_buildplan_family_id', 'build_id'), Index('idx_buildplan_plan_id', 'plan_id'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) build_id = Column(GUID, ForeignKey('build.id', ondelete="CASCADE"), nullable=False) job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False, unique=True) plan_id = Column(GUID, ForeignKey('plan.id', ondelete="CASCADE"), nullable=False) snapshot_image_id = Column(GUID, ForeignKey('snapshot_image.id', ondelete="RESTRICT"), nullable=True) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) project = relationship('Project') build = relationship('Build') job = relationship('Job') plan = relationship('Plan') snapshot_image = relationship('SnapshotImage') __repr__ = model_repr('build_id', 'job_id', 'plan_id') def __init__(self, **kwargs): super(JobPlan, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created def get_steps(self): if 'snapshot' in self.data: return map(lambda x: HistoricalImmutableStep(**x), self.data['snapshot']['steps']) return map(HistoricalImmutableStep.from_step, self.plan.steps) # TODO(dcramer): find a better place for this @classmethod def build_jobplan(cls, plan, job, snapshot_id=None): """Creates and returns a jobplan. Unless a snapshot_id is given, no snapshot will be used. This differs from the build index endpoint where the default is the current snapshot for a project. If a snapshot image is not found for a plan configured to use snapshots, a warning is given. """ from changes.models.option import ItemOption from changes.models.snapshot import SnapshotImage plan_steps = sorted(plan.steps, key=lambda x: x.order) option_item_ids = [s.id for s in plan_steps] option_item_ids.append(plan.id) options = defaultdict(dict) options_query = db.session.query( ItemOption.item_id, ItemOption.name, ItemOption.value).filter(ItemOption.item_id.in_(option_item_ids), ) for item_id, opt_name, opt_value in options_query: options[item_id][opt_name] = opt_value snapshot = { 'steps': [ HistoricalImmutableStep.from_step(s, options[s.id]).to_json() for s in plan_steps ], 'options': options[plan.id], } snapshot_image_id = None # TODO(paulruan): Remove behavior that just having a snapshot plan means # snapshot use is enabled. Just `snapshot.allow` should be sufficient. allow_snapshot = '1' == options[plan.id].get('snapshot.allow', '1') or plan.snapshot_plan if allow_snapshot and snapshot_id is not None: snapshot_image = SnapshotImage.get(plan, snapshot_id) if snapshot_image is not None: snapshot_image_id = snapshot_image.id if snapshot_image is None: logging.warning("Failed to find snapshot_image for %s's %s.", plan.project.slug, plan.label) instance = cls( plan_id=plan.id, job_id=job.id, build_id=job.build_id, project_id=job.project_id, snapshot_image_id=snapshot_image_id, data={ 'snapshot': snapshot, }, ) return instance # TODO(dcramer): this is a temporary method and should be removed once we # support more than a single job (it also should not be contained within # the model file) @classmethod def get_build_step_for_job(cls, job_id): from changes.models.project import ProjectConfigError from changes.buildsteps.lxc import LXCBuildStep jobplan = cls.query.filter(cls.job_id == job_id, ).first() if jobplan is None: return None, None if jobplan.plan.autogenerated(): job = jobplan.job try: diff = job.source.patch.diff if job.source.patch else None project_config = job.project.get_config( job.source.revision_sha, diff=diff) except ProjectConfigError: logging.error( 'Project config for project %s is not in a valid format.', job.project.slug, exc_info=True) return jobplan, None if 'bazel.targets' not in project_config: logging.error( 'Project config for project %s is missing `bazel.targets`. job: %s, revision_sha: %s, config: %s', job.project.slug, job.id, job.source.revision_sha, str(project_config), exc_info=True) return jobplan, None bazel_exclude_tags = project_config['bazel.exclude-tags'] bazel_cpus = project_config['bazel.cpus'] bazel_max_executors = project_config['bazel.max-executors'] if bazel_cpus < 1 or bazel_cpus > current_app.config[ 'MAX_CPUS_PER_EXECUTOR']: logging.error( 'Project config for project %s requests invalid number of CPUs: constraint 1 <= %d <= %d' % (job.project.slug, bazel_cpus, current_app.config['MAX_CPUS_PER_EXECUTOR'])) return jobplan, None bazel_memory = project_config['bazel.mem'] if bazel_memory < current_app.config['MIN_MEM_MB_PER_EXECUTOR'] or \ bazel_memory > current_app.config['MAX_MEM_MB_PER_EXECUTOR']: logging.error( 'Project config for project %s requests invalid memory requirements: constraint %d <= %d <= %d' % (job.project.slug, current_app.config['MIN_MEM_MB_PER_EXECUTOR'], bazel_memory, current_app.config['MAX_MEM_MB_PER_EXECUTOR'])) return jobplan, None if bazel_max_executors < 1 or bazel_max_executors > current_app.config[ 'MAX_EXECUTORS']: logging.error( 'Project config for project %s requests invalid number of executors: constraint 1 <= %d <= %d', job.project.slug, bazel_max_executors, current_app.config['MAX_EXECUTORS']) return jobplan, None additional_test_flags = project_config[ 'bazel.additional-test-flags'] for f in additional_test_flags: patterns = current_app.config[ 'BAZEL_ADDITIONAL_TEST_FLAGS_WHITELIST_REGEX'] if not any([re.match(p, f) for p in patterns]): logging.error( 'Project config for project %s contains invalid additional-test-flags %s. Allowed patterns are %s.', job.project.slug, f, patterns) return jobplan, None bazel_test_flags = current_app.config[ 'BAZEL_MANDATORY_TEST_FLAGS'] + additional_test_flags bazel_test_flags = list( OrderedDict([(b, None) for b in bazel_test_flags ])) # ensure uniqueness, preserve order # TODO(anupc): Does it make sense to expose this in project config? bazel_debug_config = current_app.config['BAZEL_DEBUG_CONFIG'] if 'prelaunch_env' not in bazel_debug_config: bazel_debug_config['prelaunch_env'] = {} vcs = job.project.repository.get_vcs() bazel_debug_config['prelaunch_env'][ 'REPO_URL'] = job.project.repository.url bazel_debug_config['prelaunch_env'][ 'REPO_NAME'] = vcs.get_repository_name( job.project.repository.url) implementation = LXCBuildStep( cluster=current_app.config['DEFAULT_CLUSTER'], commands=[ { 'script': get_bazel_setup(), 'type': 'setup' }, { 'script': sync_encap_pkgs(project_config), 'type': 'setup' }, # TODO(anupc): Make this optional { 'script': extra_setup_cmd(), 'type': 'setup' }, # TODO(anupc): Make this optional { 'script': collect_bazel_targets( collect_targets_executable=os.path.join( LXCBuildStep.custom_bin_path(), 'collect-targets'), bazel_targets=project_config['bazel.targets'], bazel_exclude_tags=bazel_exclude_tags, max_jobs=2 * bazel_cpus, bazel_test_flags=bazel_test_flags, skip_list_patterns=[job.project.get_config_path()], ), 'type': 'collect_bazel_targets', 'env': { 'VCS_CHECKOUT_TARGET_REVISION_CMD': vcs.get_buildstep_checkout_revision('master'), 'VCS_CHECKOUT_PARENT_REVISION_CMD': vcs.get_buildstep_checkout_parent_revision( 'master'), 'VCS_GET_CHANGED_FILES_CMD': vcs.get_buildstep_changed_files('master'), }, }, ], artifacts= [], # only for collect_target step, which we don't expect artifacts artifact_suffix=current_app.config['BAZEL_ARTIFACT_SUFFIX'], cpus=bazel_cpus, memory=bazel_memory, max_executors=bazel_max_executors, debug_config=bazel_debug_config, ) return jobplan, implementation steps = jobplan.get_steps() try: step = steps[0] except IndexError: return jobplan, None return jobplan, step.get_implementation()
class Build(db.Model): """ Represents the work we do (e.g. running tests) for one diff or commit (an entry in the source table) in one particular project Each Build contains many Jobs (usually linked to a JobPlan). """ __tablename__ = 'build' __table_args__ = ( Index('idx_buildfamily_project_id', 'project_id'), Index('idx_buildfamily_author_id', 'author_id'), Index('idx_buildfamily_source_id', 'source_id'), UniqueConstraint('project_id', 'number', name='unq_build_number'), ) id = Column(GUID, primary_key=True, default=uuid.uuid4) number = Column(Integer) project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False) # A unqiue identifier for a group of related Builds, such as all Builds created by a particular # action. Used primarily for aggregation in result reporting. # Note that this may be None for Builds that aren't grouped, and all such Builds should NOT # be treated as a collection. collection_id = Column(GUID) source_id = Column(GUID, ForeignKey('source.id', ondelete="CASCADE")) author_id = Column(GUID, ForeignKey('author.id', ondelete="CASCADE")) cause = Column(EnumType(Cause), nullable=False, default=Cause.unknown) # label is a short description, typically from the title of the change that triggered the build. label = Column(String(128), nullable=False) # short indicator of what is being built, typically the sha or the Phabricator revision ID like 'D90885'. target = Column(String(128)) tags = Column(ARRAY(String(16)), nullable=True) status = Column(EnumType(Status), nullable=False, default=Status.unknown) result = Column(EnumType(Result), nullable=False, default=Result.unknown) message = Column(Text) duration = Column(Integer) priority = Column(EnumType(BuildPriority), nullable=False, default=BuildPriority.default, server_default='0') date_started = Column(DateTime) date_finished = Column(DateTime) date_created = Column(DateTime, default=datetime.utcnow) date_modified = Column(DateTime, default=datetime.utcnow) data = Column(JSONEncodedDict) project = relationship('Project', innerjoin=True) source = relationship('Source', innerjoin=True) author = relationship('Author') stats = relationship('ItemStat', primaryjoin='Build.id == ItemStat.item_id', foreign_keys=[id], uselist=True) __repr__ = model_repr('label', 'target') def __init__(self, **kwargs): super(Build, self).__init__(**kwargs) if self.id is None: self.id = uuid.uuid4() if self.result is None: self.result = Result.unknown if self.status is None: self.status = Status.unknown if self.date_created is None: self.date_created = datetime.utcnow() if self.date_modified is None: self.date_modified = self.date_created if self.date_started and self.date_finished and not self.duration: self.duration = (self.date_finished - self.date_started).total_seconds() * 1000 if self.number is None and self.project: self.number = select([func.next_item_value(self.project.id.hex)])