class Landing(db.Model):
"""DEPRECATED
This model has been replaced by landoapi.models.transplant.Transplant
and has been kept around as part of data migration. In the future when
migration has been completed this may been cleaned up and dropped from
the database with another migration.
"""
__tablename__ = "landings"
id = db.Column(db.Integer, primary_key=True)
request_id = db.Column(db.Integer, unique=True)
revision_id = db.Column(db.Integer)
diff_id = db.Column(db.Integer)
active_diff_id = db.Column(db.Integer)
status = db.Column(db.Enum(LandingStatus),
nullable=False,
default=LandingStatus.aborted)
error = db.Column(db.Text(), default='')
result = db.Column(db.Text(), default='')
requester_email = db.Column(db.String(254))
tree = db.Column(db.String(128))
created_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now())
updated_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now(),
onupdate=db.func.now())
class LandingJob(Base):
"""State for a landing job."""
# The postgres enum column which this definition results in
# uses an enum type where the order matches the order of
# values at definition time. Since python `enum.Enum` is
# ordered, the resulting column will have the same order
# as the definition order of the enum. This can be relied
# on for comparisons or things like queries with ORDER BY.
status = db.Column(db.Enum(LandingJobStatus),
nullable=False,
default=LandingJobStatus.SUBMITTED)
# JSON object mapping string revision id of the form "<int>" (used because
# json keys may not be integers) to integer diff id. This is used to
# record the diff id used with each revision and make searching for
# Transplants that match a set of revisions easy (such as those
# in a stack).
# e.g.
# {
# "1001": 1221,
# "1002": 1246,
# "1003": 1412
# }
revision_to_diff_id = db.Column(JSONB, nullable=False)
# JSON array of string revision ids of the form "<int>" (used to match
# the string type of revision_to_diff_id keys) listing the order
# of the revisions in the request from most ancestral to most
# descendant.
# e.g.
# ["1001", "1002", "1003"]
revision_order = db.Column(JSONB, nullable=False)
# Text describing errors when status != LANDED.
error = db.Column(db.Text(), default="")
# Error details in a dictionary format, listing failed merges, etc...
# E.g. {
# "failed_paths": [{"path": "...", "url": "..."}],
# "reject_paths": [{"path": "...", "content": "..."}]
# }
error_breakdown = db.Column(JSONB, nullable=True)
# LDAP email of the user who requested transplant.
requester_email = db.Column(db.String(254), nullable=False)
# Lando's name for the repository.
repository_name = db.Column(db.Text(), nullable=False)
# URL of the repository revisions are to land to.
repository_url = db.Column(db.Text(), default="")
# Identifier for the most descendent commit created by this landing.
landed_commit_id = db.Column(db.Text(), default="")
# Number of attempts made to complete the job.
attempts = db.Column(db.Integer, nullable=False, default=0)
# Priority of the job. Higher values are processed first.
priority = db.Column(db.Integer, nullable=False, default=0)
# Duration of job from start to finish
duration_seconds = db.Column(db.Integer, default=0)
@property
def landing_path(self):
return [(int(r), self.revision_to_diff_id[r])
for r in self.revision_order]
@property
def head_revision(self):
"""Human-readable representation of the branch head's Phabricator revision ID.
"""
assert (
self.revision_order
), "head_revision should never be called without setting self.revision_order!"
return "D" + self.revision_order[-1]
@classmethod
def revisions_query(cls, revisions):
revisions = [str(int(r)) for r in revisions]
return cls.query.filter(
cls.revision_to_diff_id.has_any(array(revisions)))
@classmethod
def job_queue_query(cls,
repositories=None,
grace_seconds=DEFAULT_GRACE_SECONDS):
"""Return a query which selects the queued jobs.
Args:
repositories (iterable): A list of repository names to use when filtering
the landing job search query.
grace_seconds (int): Ignore landing jobs that were submitted after this
many seconds ago.
"""
applicable_statuses = (
LandingJobStatus.SUBMITTED,
LandingJobStatus.IN_PROGRESS,
LandingJobStatus.DEFERRED,
)
q = cls.query.filter(cls.status.in_(applicable_statuses))
if repositories:
q = q.filter(cls.repository_name.in_(repositories))
if grace_seconds:
now = datetime.datetime.now(datetime.timezone.utc)
grace_cutoff = now - datetime.timedelta(seconds=grace_seconds)
q = q.filter(cls.created_at < grace_cutoff)
# Any `LandingJobStatus.IN_PROGRESS` job is first and there should
# be a maximum of one (per repository). For
# `LandingJobStatus.SUBMITTED` jobs, higher priority items come first
# and then we order by creation time (older first).
q = q.order_by(cls.status.desc(), cls.priority.desc(), cls.created_at)
return q
@classmethod
def next_job_for_update_query(cls, repositories=None):
"""Return a query which selects the next job and locks the row."""
q = cls.job_queue_query(repositories=repositories)
# Returned rows should be locked for updating, this ensures the next
# job can be claimed.
q = q.with_for_update()
return q
def transition_status(self, action, commit=False, db=None, **kwargs):
"""Change the status and other applicable fields according to actions.
Args:
action (str): the action to take, e.g. "land" or "fail"
commit (bool): whether to commit the changes to the database or not
db (SQLAlchemy.db): the database to commit to
**kwargs:
Additional arguments required by each action, e.g. `message` or
`commit_id`.
"""
actions = {
LandingJobAction.LAND: {
"required_params": ["commit_id"],
"status": LandingJobStatus.LANDED,
},
LandingJobAction.FAIL: {
"required_params": ["message"],
"status": LandingJobStatus.FAILED,
},
LandingJobAction.DEFER: {
"required_params": ["message"],
"status": LandingJobStatus.DEFERRED,
},
LandingJobAction.CANCEL: {
"required_params": [],
"status": LandingJobStatus.CANCELLED,
},
}
if action not in actions:
raise ValueError(f"{action} is not a valid action")
required_params = actions[action]["required_params"]
if sorted(required_params) != sorted(kwargs.keys()):
missing_params = required_params - kwargs.keys()
raise ValueError(f"Missing {missing_params} params")
if commit and db is None:
raise ValueError("db is required when commit is set to True")
self.status = actions[action]["status"]
if action in (LandingJobAction.FAIL, LandingJobAction.DEFER):
self.error = kwargs["message"]
if action == LandingJobAction.LAND:
self.landed_commit_id = kwargs["commit_id"]
if commit:
db.session.commit()
def serialize(self):
"""Return a JSON compatible dictionary."""
return {
"id":
self.id,
"status":
self.status.value,
"landing_path": [{
"revision_id": "D{}".format(r),
"diff_id": self.revision_to_diff_id[r]
} for r in self.revision_order],
"error_breakdown":
self.error_breakdown,
"details": (self.error or self.landed_commit_id
if self.status in (LandingJobStatus.FAILED,
LandingJobStatus.CANCELLED) else
self.landed_commit_id or self.error),
"requester_email":
self.requester_email,
"tree":
self.repository_name,
"repository_url":
self.repository_url,
"created_at":
(self.created_at.astimezone(datetime.timezone.utc).isoformat()),
"updated_at":
(self.updated_at.astimezone(datetime.timezone.utc).isoformat()),
}
class Transplant(Base):
"""Represents a request to Autoland Transplant."""
__tablename__ = "transplants"
# Autoland Transplant request ID.
request_id = db.Column(db.Integer, unique=True)
status = db.Column(db.Enum(TransplantStatus),
nullable=False,
default=TransplantStatus.aborted)
# JSON object mapping string revision id of the form "<int>" (used because
# json keys may not be integers) to integer diff id. This is used to
# record the diff id used with each revision and make searching for
# Transplants that match a set of revisions easy (such as those
# in a stack).
# e.g.
# {
# "1001": 1221,
# "1002": 1246,
# "1003": 1412
# }
revision_to_diff_id = db.Column(JSONB, nullable=False)
# JSON array of string revision ids of the form "<int>" (used to match
# the string type of revision_to_diff_id keys) listing the order
# of the revisions in the request from most ancestral to most
# descendant.
# e.g.
# ["1001", "1002", "1003"]
revision_order = db.Column(JSONB, nullable=False)
# Text describing errors when not landed.
error = db.Column(db.Text(), default="")
# Revision (sha) of the head of the push.
result = db.Column(db.Text(), default="")
# LDAP email of the user who requested transplant.
requester_email = db.Column(db.String(254))
# URL of the repository revisions are to land to.
repository_url = db.Column(db.Text(), default="")
# Treestatus tree name the revisions are to land to.
tree = db.Column(db.String(128))
def update_from_transplant(self, landed, error="", result=""):
"""Set the status from pingback request."""
self.error = error
self.result = result
if not landed:
self.status = (TransplantStatus.failed
if error else TransplantStatus.submitted)
else:
self.status = TransplantStatus.landed
@property
def landing_path(self):
return [(int(r), self.revision_to_diff_id[r])
for r in self.revision_order]
@property
def head_revision(self):
"""Human-readable representation of the branch head's Phabricator revision ID.
"""
assert (
self.revision_order
), "head_revision should never be called without setting self.revision_order!"
return "D" + self.revision_order[-1]
@classmethod
def revisions_query(cls, revisions):
revisions = [str(int(r)) for r in revisions]
return cls.query.filter(
cls.revision_to_diff_id.has_any(array(revisions)))
def serialize(self):
"""Return a JSON compatible dictionary."""
return {
"id":
self.id,
"request_id":
self.request_id,
"status":
self.status.value,
"landing_path": [{
"revision_id": "D{}".format(r),
"diff_id": self.revision_to_diff_id[r]
} for r in self.revision_order],
"details":
(self.error or self.result if self.status
in (TransplantStatus.failed,
TransplantStatus.aborted) else self.result or self.error),
"requester_email":
self.requester_email,
"tree":
self.tree,
"repository_url":
self.repository_url,
"created_at":
(self.created_at.astimezone(datetime.timezone.utc).isoformat()),
"updated_at":
(self.updated_at.astimezone(datetime.timezone.utc).isoformat()),
}
class Landing(db.Model):
"""Represents the landing process in Autoland.
Landing is communicating with Autoland via TransplantClient.
Landing is communicating with Phabricator via PhabricatorClient.
Landing object might be saved to database without creation of the actual
landing in Autoland. It is done before landing request to construct
required "pingback URL" and save related Patch objects.
To update the Landing status Transplant is calling provided pingback URL.
Attributes:
id: Primary Key
request_id: Id of the request in Autoland
revision_id: Phabricator id of the revision to be landed
diff_id: Phabricator id of the diff to be landed
status: Status of the landing. Modified by `update` API
error: Text describing the error if not landed
result: Revision (sha) of push
"""
__tablename__ = "landings"
id = db.Column(db.Integer, primary_key=True)
request_id = db.Column(db.Integer, unique=True)
revision_id = db.Column(db.String(30))
diff_id = db.Column(db.Integer)
status = db.Column(db.Integer)
error = db.Column(db.String(128), default='')
result = db.Column(db.String(128))
def __init__(self,
request_id=None,
revision_id=None,
diff_id=None,
status=TRANSPLANT_JOB_PENDING):
self.request_id = request_id
self.revision_id = revision_id
self.diff_id = diff_id
self.status = status
@classmethod
def create(cls, revision_id, diff_id, phabricator_api_key=None):
"""Land revision.
A typical successful story:
* Revision and Diff are loaded from Phabricator.
* Patch is created and uploaded to S3 bucket.
* Landing object is created (without request_id)
* A request to land the patch is send to Transplant client.
* Created landing object is updated with returned `request_id`,
it is then saved and returned.
Args:
revision_id: The id of the revision to be landed
diff_id: The id of the diff to be landed
phabricator_api_key: API Key to identify in Phabricator
Returns:
A new Landing object
Raises:
RevisionNotFoundException: PhabricatorClient returned no revision
for given revision_id
LandingNotCreatedException: landing request in Transplant failed
"""
phab = PhabricatorClient(phabricator_api_key)
revision = phab.get_revision(id=revision_id)
if not revision:
raise RevisionNotFoundException(revision_id)
repo = phab.get_revision_repo(revision)
# Save landing to make sure we've got the callback URL.
landing = cls(revision_id=revision_id, diff_id=diff_id)
landing.save()
patch = Patch(landing.id, revision, diff_id)
patch.upload(phab)
# Define the pingback URL with the port.
callback = '{host_url}/landings/{id}/update'.format(
host_url=current_app.config['PINGBACK_HOST_URL'], id=landing.id)
trans = TransplantClient()
# The LDAP username used here has to be the username of the patch
# pusher (the person who pushed the 'Land it!' button).
# FIXME: change [email protected] to the real data retrieved
# from Auth0 userinfo
request_id = trans.land('*****@*****.**', patch.s3_url,
repo['uri'], callback)
if not request_id:
raise LandingNotCreatedException
landing.request_id = request_id
landing.status = TRANSPLANT_JOB_STARTED
landing.save()
logger.info(
{
'revision_id': revision_id,
'landing_id': landing.id,
'msg': 'landing created for revision'
}, 'landing.success')
return landing
def save(self):
"""Save objects in storage."""
if not self.id:
db.session.add(self)
return db.session.commit()
def __repr__(self):
return '<Landing: %s>' % self.id
def serialize(self):
"""Serialize to JSON compatible dictionary."""
return {
'id': self.id,
'revision_id': self.revision_id,
'request_id': self.request_id,
'diff_id': self.diff_id,
'status': self.status,
'error_msg': self.error,
'result': self.result
}
class Transplant(db.Model):
"""Represents a request to Autoland Transplant."""
__tablename__ = "transplants"
# Internal request ID.
id = db.Column(db.Integer, primary_key=True)
# Autoland Transplant request ID.
request_id = db.Column(db.Integer, unique=True)
status = db.Column(db.Enum(TransplantStatus),
nullable=False,
default=TransplantStatus.aborted)
created_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now())
updated_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now(),
onupdate=db.func.now())
# JSON object mapping string revision id of the form "<int>" (used because
# json keys may not be integers) to integer diff id. This is used to
# record the diff id used with each revision and make searching for
# Transplants that match a set of revisions easy (such as those
# in a stack).
# e.g.
# {
# "1001": 1221,
# "1002": 1246,
# "1003": 1412
# }
revision_to_diff_id = db.Column(JSONB, nullable=False)
# JSON array of string revision ids of the form "<int>" (used to match
# the string type of revision_to_diff_id keys) listing the order
# of the revisions in the request from most ancestral to most
# descendant.
# e.g.
# ["1001", "1002", "1003"]
revision_order = db.Column(JSONB, nullable=False)
# Text describing errors when not landed.
error = db.Column(db.Text(), default='')
# Revision (sha) of the head of the push.
result = db.Column(db.Text(), default='')
# LDAP email of the user who requested transplant.
requester_email = db.Column(db.String(254))
# URL of the repository revisions are to land to.
repository_url = db.Column(db.Text(), default='')
# Treestatus tree name the revisions are to land to.
tree = db.Column(db.String(128))
def __repr__(self):
return '<Transplant: %s>' % self.id
def update_from_transplant(self, landed, error='', result=''):
"""Set the status from pingback request."""
self.error = error
self.result = result
if not landed:
self.status = (TransplantStatus.failed
if error else TransplantStatus.submitted)
else:
self.status = TransplantStatus.landed
@property
def landing_path(self):
return [(int(r), self.revision_to_diff_id[r])
for r in self.revision_order]
@classmethod
def revisions_query(cls, revisions):
revisions = [str(int(r)) for r in revisions]
return cls.query.filter(
cls.revision_to_diff_id.has_any(array(revisions)))
@classmethod
def is_revision_submitted(cls, revision_id):
"""Check if revision is successfully submitted.
Args:
revision_id: The integer id of the revision.
Returns:
Transplant object or False if not submitted.
"""
transplants = cls.revisions_query(
[revision_id]).filter_by(status=TransplantStatus.submitted).all()
if not transplants:
return False
return transplants[0]
@classmethod
def legacy_latest_landed(cls, revision_id):
"""DEPRECATED Return the latest Landing that is landed, or None.
Args:
revision_id: The integer id of the revision.
Returns:
Latest transplant object with status landed, or None if
none exist.
"""
return cls.revisions_query(
[revision_id]).filter_by(status=TransplantStatus.landed).order_by(
cls.updated_at.desc()).first()
def serialize(self):
"""Return a JSON compatible dictionary."""
return {
'id': self.id,
'request_id': self.request_id,
'status': self.status.value,
'landing_path': [
{
'revision_id': 'D{}'.format(r),
'diff_id': self.revision_to_diff_id[r],
} for r in self.revision_order
],
'details': (
self.error or self.result
if self.status in (
TransplantStatus.failed,
TransplantStatus.aborted
)
else self.result or self.error
),
'requester_email': self.requester_email,
'tree': self.tree,
'repository_url': self.repository_url,
'created_at': (
self.created_at.astimezone(datetime.timezone.utc).isoformat()
),
'updated_at': (
self.updated_at.astimezone(datetime.timezone.utc).isoformat()
),
} # yapf: disable
def legacy_serialize(self):
"""DEPRECATED Serialize to JSON compatible dictionary."""
revision_id = None
diff_id = None
if self.revision_order is not None:
revision_id = self.revision_order[-1]
if (revision_id is not None and self.revision_to_diff_id is not None):
diff_id = self.revision_to_diff_id.get(revision_id)
return {
'id': self.id,
'revision_id': 'D{}'.format(revision_id),
'request_id': self.request_id,
'diff_id': diff_id,
'active_diff_id': diff_id,
'status': self.status.value,
'error_msg': self.error,
'result': self.result,
'requester_email': self.requester_email,
'tree': self.tree,
'tree_url': self.repository_url or '',
'created_at': (
self.created_at.astimezone(datetime.timezone.utc).isoformat()
),
'updated_at': (
self.updated_at.astimezone(datetime.timezone.utc).isoformat()
),
} # yapf: disable
class Landing(db.Model):
"""Represents the landing process in Autoland.
Landing is communicating with Autoland via TransplantClient.
Landing is communicating with Phabricator via PhabricatorClient.
Landing object might be saved to database without creation of the actual
landing in Autoland. It is done before landing request to construct
required "pingback URL" and save related Patch objects.
To update the Landing status Transplant is calling provided pingback URL.
Active Diff Id is stored on creation if it is different than diff_id.
Attributes:
id: Primary Key
request_id: Id of the request in Autoland
revision_id: Phabricator id of the revision to be landed
diff_id: Phabricator id of the diff to be landed
active_diff_id: Phabricator id of the diff active at the moment of
landing
status: Status of the landing. Modified by `update` API
error: Text describing the error if not landed
result: Revision (sha) of push
requester_email: The email address of the requester of the landing.
tree: The treestatus tree name the revision is to land to.
created_at: DateTime of the creation
updated_at: DateTime of the last save
"""
__tablename__ = "landings"
id = db.Column(db.Integer, primary_key=True)
request_id = db.Column(db.Integer, unique=True)
revision_id = db.Column(db.Integer)
diff_id = db.Column(db.Integer)
active_diff_id = db.Column(db.Integer)
status = db.Column(db.Enum(LandingStatus),
nullable=False,
default=LandingStatus.aborted)
error = db.Column(db.Text(), default='')
result = db.Column(db.Text(), default='')
requester_email = db.Column(db.String(254))
tree = db.Column(db.String(128))
created_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now())
updated_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=db.func.now(),
onupdate=db.func.now())
@classmethod
def is_revision_submitted(cls, revision_id):
"""Check if revision is successfully submitted.
Args:
revision_id: The integer id of the revision.
Returns:
Landed Revision object or False if not submitted.
"""
landings = cls.query.filter(
cls.revision_id == revision_id,
cls.status == LandingStatus.submitted).all()
if not landings:
return False
return landings[0]
@classmethod
def latest_landed(cls, revision_id):
"""Return the latest Landing that is landed, or None.
Args:
revision_id: The integer id of the revision.
Returns:
Latest landing object with status landed, or None if
none exist.
"""
return cls.query.filter_by(revision_id=revision_id,
status=LandingStatus.landed).order_by(
cls.updated_at.desc()).first()
def __repr__(self):
return '<Landing: %s>' % self.id
def serialize(self):
"""Serialize to JSON compatible dictionary."""
return {
'id': self.id,
'revision_id': 'D{}'.format(self.revision_id),
'request_id': self.request_id,
'diff_id': self.diff_id,
'active_diff_id': self.active_diff_id,
'status': self.status.value,
'error_msg': self.error,
'result': self.result,
'requester_email': self.requester_email,
'tree': self.tree,
'created_at': (
self.created_at.astimezone(datetime.timezone.utc).isoformat()
),
'updated_at': (
self.updated_at.astimezone(datetime.timezone.utc).isoformat()
),
} # yapf: disable
def update_from_transplant(self, landed, error='', result=''):
"""Set the status from pingback request."""
self.error = error
self.result = result
if not landed:
self.status = (LandingStatus.failed
if error else LandingStatus.submitted)
else:
self.status = LandingStatus.landed