def _sync_project_admin_users(project_admin_mapping):
# type: (Dict[str, Set[str]]) -> None
"""This synchronizes the Changes user database so that only people
in `project_admin_mapping` are project admins, and that they are
admins only for the projects they have permissions to. Note that
if a user who should be a project admin does not exist in the Changes
database, the user is created.
Args:
project_admin_mapping (Dict[str, Set[str]]): The mapping from
user emails to project patterns
"""
args = [~User.project_permissions.is_(None)]
if len(project_admin_mapping) > 0:
args.append(~User.email.in_(project_admin_mapping.keys()))
User.query.filter(*args).update({'project_permissions': None},
synchronize_session=False)
for email, project_permissions in project_admin_mapping.iteritems():
create_or_update(
User,
where={
'email': email,
},
values={'project_permissions': list(project_permissions)})
db.session.commit()
def _sync_admin_users(admin_emails):
# type: (Set[str]) -> None
"""Take a look at the Changes user database. Every user with email in
`admin_emails` should become a Changes admin, and every user already
an admin whose email is not in `admin_emails` will have their
admin privileges revoked. Note that if a user who should be an admin
does not exist in the Changes database, the user is created.
Args:
admin_emails (iterable[basestring]): an iterable of usernames of
people who should be admin.
"""
# revoke access for people who should not have admin access
assert len(admin_emails) > 0
User.query.filter(
~User.email.in_(admin_emails),
User.is_admin.is_(True),
).update({
'is_admin': False,
}, synchronize_session=False)
# give access for people who should have access
for email in admin_emails:
create_or_update(User,
where={
'email': email,
},
values={
'is_admin': True,
})
db.session.commit()
def post(self, snapshot_id):
snapshot = Snapshot.query.get(snapshot_id)
if snapshot is None:
return '', 404
args = self.parser.parse_args()
if args.status:
snapshot.status = SnapshotStatus[args.status]
if args.set_current and snapshot.status != SnapshotStatus.active:
return '{"error": "Cannot set inactive current snapshot"}', 400
db.session.add(snapshot)
db.session.commit()
if args.set_current:
# TODO(adegtiar): improve logic for picking current snapshot.
create_or_update(ProjectOption, where={
'project': snapshot.project,
'name': 'snapshot.current',
}, values={
'value': snapshot.id.hex,
})
return self.respond(snapshot)
def post(self, project_id):
project = self._get_project(project_id)
if project is None:
return error("Project not found", http_code=404)
args = self.parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
# If we're rolling back a snapshot, take note.
if name == 'snapshot.current':
current = Snapshot.get_current(project.id)
if current:
# If value is empty, we're deactivating a snapshot without replacing it,
# and that's a downgrade too.
replacement = value and Snapshot.query.get(value)
if not replacement or replacement.date_created < current.date_created:
_report_snapshot_downgrade(project)
create_or_update(ProjectOption, where={
'project': project,
'name': name,
}, values={
'value': value,
})
return self.respond({})
def post(self, project_id):
project = self._get_project(project_id)
if project is None:
return error("Project not found", http_code=404)
args = self.parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
# If we're rolling back a snapshot, take note.
if name == 'snapshot.current':
current = Snapshot.get_current(project.id)
if current:
# If value is empty, we're deactivating a snapshot without replacing it,
# and that's a downgrade too.
replacement = value and Snapshot.query.get(value)
if not replacement or replacement.date_created < current.date_created:
_report_snapshot_downgrade(project)
create_or_update(ProjectOption,
where={
'project': project,
'name': name,
},
values={
'value': value,
})
return self.respond({})
def _record_test_rerun_counts(self, test_list):
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_rerun_count',
}, values={
'value': sum(1 for t in test_list if t.reruns),
})
def post(self, snapshot_id):
snapshot = Snapshot.query.get(snapshot_id)
if snapshot is None:
return '', 404
args = self.parser.parse_args()
if args.status:
snapshot.status = SnapshotStatus[args.status]
if args.set_current and snapshot.status != SnapshotStatus.active:
return '{"error": "Cannot set inactive current snapshot"}', 400
db.session.add(snapshot)
db.session.commit()
if args.set_current:
# TODO(adegtiar): improve logic for picking current snapshot.
create_or_update(ProjectOption,
where={
'project': snapshot.project,
'name': 'snapshot.current',
},
values={
'value': snapshot.id.hex,
})
return self.respond(snapshot)
def post(self, step_id):
step = Step.query.get(step_id)
if step is None:
return error("step not found", http_code=404)
args = self.parser.parse_args()
if args.implementation is not None:
step.implementation = args.implementation
if args.data is not None:
try:
data = json.loads(args.data)
except ValueError as e:
return error("invalid JSON: %s" % e)
if not isinstance(data, dict):
return error("data must be a JSON mapping")
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
return error("unable to load build step implementation")
try:
# XXX(dcramer): It's important that we deepcopy data so any
# mutations within the BuildStep don't propagate into the db
impl_cls(**deepcopy(data))
except Exception as exc:
return error(
"unable to create build step mapping provided data: %s" %
exc)
step.data = data
if args.order is not None:
step.order = args.order
step.date_modified = datetime.utcnow()
db.session.add(step)
plan = step.plan
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption,
where={
'item_id': step.id,
'name': name,
},
values={
'value': value,
})
db.session.commit()
return self.respond(step)
def _record_test_counts(self, test_list):
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_count',
}, values={
'value': len(test_list),
})
db.session.commit()
def _record_test_failures(self, test_list):
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_failures',
}, values={
'value': sum(t.result == Result.failed for t in test_list),
})
db.session.commit()
def post(self, repository_id):
repo = Repository.query.get(repository_id)
if repo is None:
return '', 404
args = self.parser.parse_args()
if args.url:
repo.url = args.url
if args.backend:
repo.backend = RepositoryBackend[args.backend]
needs_import = False
if args.status == 'inactive':
repo.status = RepositoryStatus.inactive
elif args.status == 'active' and repo.status == RepositoryStatus.inactive:
repo.status = RepositoryStatus.active
needs_import = True
db.session.add(repo)
for name in OPTION_DEFAULTS.keys():
value = args[name]
if value is None:
continue
# special case phabricator.callsign since we can't enforce a unique
# constraint
if name == 'phabricator.callsign':
existing = ItemOption.query.filter(
ItemOption.item_id != repo.id,
ItemOption.name == name,
ItemOption.value == value,
).first()
if existing:
return '{"error": "A repository already exists with the given Phabricator callsign"}', 400
create_or_update(ItemOption,
where={
'item_id': repo.id,
'name': name,
},
values={
'value': value,
})
db.session.commit()
if needs_import:
import_repo.delay_if_needed(
repo_id=repo.id.hex,
task_id=repo.id.hex,
)
context = self.serialize(repo)
context['options'] = self._get_options(repo)
return self.respond(context, serialize=False)
def post(self, step_id):
step = Step.query.get(step_id)
if step is None:
return {"message": "step not found"}, 404
args = self.parser.parse_args()
if args.implementation is not None:
step.implementation = args.implementation
if args.data is not None:
data = json.loads(args.data)
if not isinstance(data, dict):
return {"message": "data must be a JSON mapping"}, 400
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
return {
"message": "unable to load build step implementation"
}, 400
try:
impl_cls(**data)
except Exception:
return {
"message":
"unable to create build step mapping provided data"
}, 400
step.data = data
if args.order is not None:
step.order = args.order
step.date_modified = datetime.utcnow()
db.session.add(step)
plan = step.plan
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption,
where={
'item_id': step.id,
'name': name,
},
values={
'value': value,
})
db.session.commit()
return self.serialize(step), 200
def _record_test_duration(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_duration',
}, values={
'value': db.session.query(func.coalesce(func.sum(TestCase.duration), 0)).filter(
TestCase.step_id == step.id,
).as_scalar(),
})
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return error("plan not found", http_code=404)
args = self.parser.parse_args()
step = Step(
plan=plan,
order=args.order,
implementation=args.implementation,
)
try:
data = json.loads(args.data)
except ValueError as e:
db.session.rollback()
return error("invalid JSON: %s" % e)
if not isinstance(data, dict):
db.session.rollback()
return error("data must be a JSON mapping")
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
db.session.rollback()
return error("unable to load build step implementation")
try:
# XXX(dcramer): It's important that we deepcopy data so any
# mutations within the BuildStep don't propagate into the db
impl_cls(**deepcopy(data))
except Exception as exc:
db.session.rollback()
return error("unable to create build step provided data: %s" % exc)
step.data = data
step.order = args.order
db.session.add(step)
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption,
where={
'item_id': step.id,
'name': name,
},
values={
'value': value,
})
return self.serialize(step), 201
def _record_test_duration(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_duration',
}, values={
'value': db.session.query(func.coalesce(func.sum(TestCase.duration), 0)).filter(
TestCase.step_id == step.id,
).as_scalar(),
})
def post(self, repository_id):
repo = Repository.query.get(repository_id)
if repo is None:
return '', 404
args = self.parser.parse_args()
if args.url:
repo.url = args.url
if args.backend:
repo.backend = RepositoryBackend[args.backend]
needs_import = False
if args.status == 'inactive':
repo.status = RepositoryStatus.inactive
elif args.status == 'active' and repo.status == RepositoryStatus.inactive:
repo.status = RepositoryStatus.active
needs_import = True
db.session.add(repo)
for name in OPTION_DEFAULTS.keys():
value = args[name]
if value is None:
continue
# special case phabricator.callsign since we can't enforce a unique
# constraint
if name == 'phabricator.callsign':
existing = ItemOption.query.filter(
ItemOption.item_id != repo.id,
ItemOption.name == name,
ItemOption.value == value,
).first()
if existing:
return '{"error": "A repository already exists with the given Phabricator callsign"}', 400
create_or_update(ItemOption, where={
'item_id': repo.id,
'name': name,
}, values={
'value': value,
})
db.session.commit()
if needs_import:
import_repo.delay_if_needed(
repo_id=repo.id.hex,
task_id=repo.id.hex,
)
context = self.serialize(repo)
context['options'] = self._get_options(repo)
return self.respond(context, serialize=False)
def post(self, step_id):
step = Step.query.get(step_id)
if step is None:
return error("step not found", http_code=404)
args = self.parser.parse_args()
if args.implementation is not None:
step.implementation = args.implementation
if args.data is not None:
try:
data = json.loads(args.data)
except ValueError as e:
return error("invalid JSON: %s" % e)
if not isinstance(data, dict):
return error("data must be a JSON mapping")
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
return error("unable to load build step implementation")
try:
# XXX(dcramer): It's important that we deepcopy data so any
# mutations within the BuildStep don't propagate into the db
impl_cls(**deepcopy(data))
except Exception as exc:
return error("unable to create build step mapping provided data: %s" % exc)
step.data = data
if args.order is not None:
step.order = args.order
step.date_modified = datetime.utcnow()
db.session.add(step)
plan = step.plan
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': step.id,
'name': name,
}, values={
'value': value,
})
db.session.commit()
return self.serialize(step), 200
def _record_test_counts(self, test_list):
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_count',
}, values={
'value': db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == self.step.id,
).as_scalar(),
})
db.session.commit()
def _record_test_rerun_counts(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_rerun_count',
}, values={
'value': db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == step.id,
TestCase.reruns > 0,
).as_scalar(),
})
def _record_test_rerun_counts(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_rerun_count',
}, values={
'value': db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == step.id,
TestCase.reruns > 0,
).as_scalar(),
})
def _record_test_failures(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_failures',
}, values={
'value': db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == step.id,
TestCase.result == Result.failed,
).as_scalar(),
})
db.session.commit()
def _record_test_failures(step):
create_or_update(ItemStat, where={
'item_id': step.id,
'name': 'test_failures',
}, values={
'value': db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == step.id,
TestCase.result == Result.failed,
).as_scalar(),
})
db.session.commit()
def _record_test_duration(self, test_list):
if test_list:
local_test_duration = sum(t.duration or 0 for t in test_list)
else:
local_test_duration = 0
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_duration',
}, values={
'value': local_test_duration,
})
def _record_test_counts(self, test_list):
create_or_update(
ItemStat,
where={
'item_id': self.step.id,
'name': 'test_count',
},
values={
'value':
db.session.query(func.count(TestCase.id)).filter(
TestCase.step_id == self.step.id, ).as_scalar(),
})
db.session.commit()
def post(self, step_id):
step = Step.query.get(step_id)
if step is None:
return {"message": "step not found"}, 404
args = self.parser.parse_args()
if args.implementation is not None:
step.implementation = args.implementation
if args.data is not None:
data = json.loads(args.data)
if not isinstance(data, dict):
return {"message": "data must be a JSON mapping"}, 400
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
return {"message": "unable to load build step implementation"}, 400
try:
impl_cls(**data)
except Exception:
return {"message": "unable to create build step mapping provided data"}, 400
step.data = data
if args.order is not None:
step.order = args.order
step.date_modified = datetime.utcnow()
db.session.add(step)
plan = step.plan
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': step.id,
'name': name,
}, values={
'value': value,
})
db.session.commit()
return self.serialize(step), 200
def post(self):
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(SystemOption, where={
'name': name,
}, values={
'value': value,
})
return '', 200
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return {"message": "plan not found"}, 404
args = self.parser.parse_args()
step = Step(
plan=plan,
order=args.order,
implementation=args.implementation,
)
data = json.loads(args.data)
if not isinstance(data, dict):
return {"message": "data must be a JSON mapping"}, 400
impl_cls = step.get_implementation(load=False)
if impl_cls is None:
return {"message": "unable to load build step implementation"}, 400
try:
# XXX(dcramer): It's important that we deepcopy data so any
# mutations within the BuildStep don't propagate into the db
impl_cls(**deepcopy(data))
except Exception:
return {"message": "unable to create build step provided data"}, 400
step.data = data
step.order = args.order
db.session.add(step)
plan.date_modified = step.date_modified
db.session.add(plan)
for name in STEP_OPTIONS.keys():
value = args.get(name)
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': step.id,
'name': name,
}, values={
'value': value,
})
return self.serialize(step), 201
def save(self, repository):
author = self._get_author(self.author)
if self.author == self.committer:
committer = author
else:
committer = self._get_author(self.committer)
revision, created = create_or_update(Revision, where={
'repository': repository,
'sha': self.id,
}, values={
'author': author,
'committer': committer,
'message': self.message,
'parents': self.parents,
'branches': self.branches,
'date_created': self.author_date,
'date_committed': self.committer_date,
})
# we also want to create a source for this item as it's the canonical
# representation in the UI
try_create(Source, {
'revision_sha': self.id,
'repository': repository,
})
return (revision, created)
def save(self, repository):
author = self._get_author(self.author)
if self.author == self.committer:
committer = author
else:
committer = self._get_author(self.committer)
revision, created = create_or_update(Revision, where={
'repository': repository,
'sha': self.id,
}, values={
'author': author,
'committer': committer,
'message': self.message,
'parents': self.parents,
'branches': self.branches,
'date_created': self.author_date,
'date_committed': self.committer_date,
})
# This call is relatively expensive - only do if necessary.
if created:
vcs = repository.get_vcs()
if vcs:
revision.patch_hash = vcs.get_patch_hash(self.id)
# we also want to create a source for this item as it's the canonical
# representation in the UI
source = try_create(Source, {
'revision_sha': self.id,
'repository': repository,
})
return (revision, created, source)
def _sync_artifact_as_log(self, artifact):
jobstep = artifact.step
job = artifact.job
logsource, created = get_or_create(LogSource,
where={
'name':
artifact.data['displayPath'],
'job': job,
'step': jobstep,
},
defaults={
'job': job,
'project': job.project,
'date_created':
job.date_started,
})
offset = 0
with closing(self.fetch_artifact(jobstep, artifact.data)) as resp:
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk,
where={
'source': logsource,
'offset': offset,
},
values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
def save(self, repository):
author = self._get_author(self.author)
if self.author == self.committer:
committer = author
else:
committer = self._get_author(self.committer)
revision, created = create_or_update(Revision,
where={
'repository': repository,
'sha': self.id,
},
values={
'author':
author,
'committer':
committer,
'message':
self.message,
'parents':
self.parents,
'branches':
self.branches,
'date_created':
self.author_date,
'date_committed':
self.committer_date,
})
return (revision, created)
def _sync_artifact_as_log(self, artifact):
jobstep = artifact.step
job = artifact.job
logsource, created = get_or_create(LogSource, where={
'name': artifact.data['displayPath'],
'job': job,
'step': jobstep,
}, defaults={
'job': job,
'project': job.project,
'date_created': job.date_started,
})
offset = 0
with closing(self.fetch_artifact(jobstep, artifact.data)) as resp:
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
def post(self):
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(SystemOption,
where={
'name': name,
},
values={
'value': value,
})
return '', 200
def save(self, repository):
author = self._get_author(self.author)
if self.author == self.committer:
committer = author
else:
committer = self._get_author(self.committer)
revision, created = create_or_update(Revision, where={
'repository': repository,
'sha': self.id,
}, values={
'author': author,
'committer': committer,
'message': self.message,
'parents': self.parents,
'branches': self.branches,
'date_created': self.author_date,
'date_committed': self.committer_date,
})
# we also want to create a source for this item as it's the canonical
# representation in the UI
source = try_create(Source, {
'revision_sha': self.id,
'repository': repository,
})
return (revision, created, source)
def _sync_artifact_as_log(self, jobstep, job_name, build_no, artifact):
job = jobstep.job
logsource, created = get_or_create(LogSource, where={
'name': artifact['displayPath'],
'job': job,
'step': jobstep,
}, defaults={
'project': job.project,
'date_created': job.date_started,
})
url = '{base}/job/{job}/{build}/artifact/{artifact}'.format(
base=self.base_url, job=job_name,
build=build_no, artifact=artifact['relativePath'],
)
offset = 0
resp = requests.get(url, stream=True, timeout=15)
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)
def _record_test_rerun_counts(self, test_list):
job = self.step.job
rerun_count = db.session.query(func.count(TestCase.id)).filter(
TestCase.job_id == job.id,
TestCase.reruns > 0,
).as_scalar()
create_or_update(ItemStat,
where={
'item_id': self.step.id,
'name': 'test_rerun_count',
},
values={
'value': sum(1 for t in test_list if t.reruns),
})
create_or_update(ItemStat,
where={
'item_id': job.id,
'name': 'test_rerun_count',
},
values={
'value': rerun_count,
})
instance = try_create(ItemStat,
where={
'item_id': job.build_id,
'name': 'test_rerun_count',
},
defaults={'value': rerun_count})
if not instance:
ItemStat.query.filter(
ItemStat.item_id == job.build_id,
ItemStat.name == 'test_rerun_count',
).update(
{
'value':
select([func.sum(ItemStat.value)]).where(
and_(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id.in_(
select([Job.id]).where(
Job.build_id == job.build_id, )))),
},
synchronize_session=False)
def _record_test_duration(self, test_list):
job = self.step.job
test_duration = db.session.query(func.sum(TestCase.duration)).filter(
TestCase.job_id == job.id, ).as_scalar()
create_or_update(ItemStat,
where={
'item_id': self.step.id,
'name': 'test_duration',
},
values={
'value': sum(t.duration for t in test_list),
})
create_or_update(ItemStat,
where={
'item_id': job.id,
'name': 'test_duration',
},
values={
'value': test_duration,
})
instance = try_create(ItemStat,
where={
'item_id': job.build_id,
'name': 'test_duration',
},
defaults={'value': test_duration})
if not instance:
ItemStat.query.filter(
ItemStat.item_id == job.build_id,
ItemStat.name == 'test_duration',
).update(
{
'value':
select([func.sum(ItemStat.value)]).where(
and_(
ItemStat.name == 'test_duration',
ItemStat.item_id.in_(
select([Job.id]).where(
Job.build_id == job.build_id, )))),
},
synchronize_session=False)
def _sync_log(self, jobstep, name, job_name, build_no):
job = jobstep.job
logsource, created = get_or_create(LogSource, where={
'name': name,
'step': jobstep,
}, defaults={
'job': job,
'project': jobstep.project,
'date_created': jobstep.date_started,
})
if created:
offset = 0
else:
offset = jobstep.data.get('log_offset', 0)
url = '{base}/job/{job}/{build}/logText/progressiveText/'.format(
base=jobstep.data['master'],
job=job_name,
build=build_no,
)
session = self.http_session
with closing(session.get(url, params={'start': offset}, stream=True, timeout=15)) as resp:
log_length = int(resp.headers['X-Text-Size'])
# When you request an offset that doesnt exist in the build log, Jenkins
# will instead return the entire log. Jenkins also seems to provide us
# with X-Text-Size which indicates the total size of the log
if offset > log_length:
return
# XXX: requests doesnt seem to guarantee chunk_size, so we force it
# with our own helper
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
# Jenkins will suggest to us that there is more data when the job has
# yet to complete
has_more = resp.headers.get('X-More-Data') == 'true'
# We **must** track the log offset externally as Jenkins embeds encoded
# links and we cant accurately predict the next `start` param.
jobstep.data['log_offset'] = log_length
db.session.add(jobstep)
return True if has_more else None
def _sync_log(self, jobstep, name, job_name, build_no):
job = jobstep.job
# TODO(dcramer): this doesnt handle concurrency
logsource, created = get_or_create(LogSource, where={
'name': name,
'job': job,
}, defaults={
'step': jobstep,
'project': jobstep.project,
'date_created': jobstep.date_started,
})
if created:
offset = 0
else:
offset = jobstep.data.get('log_offset', 0)
url = '{base}/job/{job}/{build}/logText/progressiveHtml/'.format(
base=self.base_url,
job=job_name,
build=build_no,
)
resp = requests.get(
url, params={'start': offset}, stream=True, timeout=15)
log_length = int(resp.headers['X-Text-Size'])
# When you request an offset that doesnt exist in the build log, Jenkins
# will instead return the entire log. Jenkins also seems to provide us
# with X-Text-Size which indicates the total size of the log
if offset > log_length:
return
iterator = resp.iter_content()
# XXX: requests doesnt seem to guarantee chunk_size, so we force it
# with our own helper
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk, where={
'source': logsource,
'offset': offset,
}, values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
publish_logchunk_update(chunk)
# We **must** track the log offset externally as Jenkins embeds encoded
# links and we cant accurately predict the next `start` param.
jobstep.data['log_offset'] = log_length
db.session.add(jobstep)
# Jenkins will suggest to us that there is more data when the job has
# yet to complete
return True if resp.headers.get('X-More-Data') == 'true' else None
def post(self, project_id):
project = self._get_project(project_id)
if project is None:
return '', 404
args = self.parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ProjectOption, where={
'project': project,
'name': name,
}, values={
'value': value,
})
return '', 200
def post(self):
user = get_current_user()
if user is None:
return error("User not found", http_code=404)
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': user.id,
'name': name,
}, values={
'value': value,
})
return self.respond({})
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return error("Plan not found", http_code=404)
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': plan.id,
'name': name,
}, values={
'value': value,
})
return self.respond({})
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return '', 404
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': plan.id,
'name': name,
}, values={
'value': value,
})
return '', 200
def post(self):
user = get_current_user()
if user is None:
return '', 404
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': user.id,
'name': name,
}, values={
'value': value,
})
return '', 200
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return '', 404
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption, where={
'item_id': plan.id,
'name': name,
}, values={
'value': value,
})
return '', 200
def _record_test_rerun_counts(self, test_list):
job = self.step.job
rerun_count = db.session.query(func.count(TestCase.id)).filter(
TestCase.job_id == job.id,
TestCase.reruns > 0,
).as_scalar()
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_rerun_count',
}, values={
'value': sum(1 for t in test_list if t.reruns),
})
create_or_update(ItemStat, where={
'item_id': job.id,
'name': 'test_rerun_count',
}, values={
'value': rerun_count,
})
instance = try_create(ItemStat, where={
'item_id': job.build_id,
'name': 'test_rerun_count',
}, defaults={
'value': rerun_count
})
if not instance:
ItemStat.query.filter(
ItemStat.item_id == job.build_id,
ItemStat.name == 'test_rerun_count',
).update({
'value': select([func.sum(ItemStat.value)]).where(
and_(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id.in_(select([Job.id]).where(
Job.build_id == job.build_id,
))
)
),
}, synchronize_session=False)
def _record_test_duration(self, test_list):
job = self.step.job
test_duration = db.session.query(func.sum(TestCase.duration)).filter(
TestCase.job_id == job.id,
).as_scalar()
create_or_update(ItemStat, where={
'item_id': self.step.id,
'name': 'test_duration',
}, values={
'value': sum(t.duration for t in test_list),
})
create_or_update(ItemStat, where={
'item_id': job.id,
'name': 'test_duration',
}, values={
'value': test_duration,
})
instance = try_create(ItemStat, where={
'item_id': job.build_id,
'name': 'test_duration',
}, defaults={
'value': test_duration
})
if not instance:
ItemStat.query.filter(
ItemStat.item_id == job.build_id,
ItemStat.name == 'test_duration',
).update({
'value': select([func.sum(ItemStat.value)]).where(
and_(
ItemStat.name == 'test_duration',
ItemStat.item_id.in_(select([Job.id]).where(
Job.build_id == job.build_id,
))
)
),
}, synchronize_session=False)
def post(self, plan_id):
plan = Plan.query.get(plan_id)
if plan is None:
return error("Plan not found", http_code=404)
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption,
where={
'item_id': plan.id,
'name': name,
},
values={
'value': value,
})
return self.respond({})
def post(self):
user = get_current_user()
if user is None:
return error("User not found", http_code=404)
args = self.post_parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
create_or_update(ItemOption,
where={
'item_id': user.id,
'name': name,
},
values={
'value': value,
})
return self.respond({})
def post(self, project_id):
project = self._get_project(project_id)
if project is None:
return "", 404
args = self.parser.parse_args()
for name, value in args.iteritems():
if value is None:
continue
# If we're rolling back a snapshot, take note.
if name == "snapshot.current":
current = Snapshot.get_current(project.id)
if current:
replacement = Snapshot.query.get(value)
if replacement.date_created < current.date_created:
_report_snapshot_downgrade(project)
create_or_update(ProjectOption, where={"project": project, "name": name}, values={"value": value})
return "", 200
def _set_latest_green_build_for_each_branch(build, source, vcs):
project = build.project
for branch in source.revision.branches:
current_latest_green_build = LatestGreenBuild.query.options(
joinedload('build').joinedload('source')).filter(
LatestGreenBuild.project_id == project.id,
LatestGreenBuild.branch == branch).first()
if current_latest_green_build:
child_in_question = source.revision_sha
parent_in_question = current_latest_green_build.build.source.revision_sha
try:
if not vcs.is_child_parent(
child_in_question=child_in_question,
parent_in_question=parent_in_question):
return
except UnknownChildRevision:
# The child_in_question is an unknown SHA. This shouldn't happen.
logging.exception(
"Child SHA is missing from the VCS. This is bad news and "
"shouldn't happen. (parent=%s, child=%s)",
parent_in_question, child_in_question)
return
except UnknownParentRevision:
# The parent_in_question is an unknown SHA. Assume it was
# deleted and forgotten VCS, and set the new green build to
# the child_in_question anyway.
logging.warning(
"Parent SHA is missing from the VCS. Assume it was deleted "
"and the new build is legit.",
extra={
'data': {
'repository_url': source.revision.repository.url,
'branch': branch,
'parent': parent_in_question,
'child': child_in_question,
},
})
# switch latest_green_build to this sha
green_build, _ = create_or_update(LatestGreenBuild,
where={
'project_id': project.id,
'branch': branch,
},
values={
'build': build,
})
def _set_latest_green_build_for_each_branch(build, source, vcs):
project = build.project
for branch in source.revision.branches:
current_latest_green_build = LatestGreenBuild.query.options(
joinedload('build').joinedload('source')
).filter(
LatestGreenBuild.project_id == project.id,
LatestGreenBuild.branch == branch).first()
if current_latest_green_build:
child_in_question = source.revision_sha
parent_in_question = current_latest_green_build.build.source.revision_sha
try:
if not vcs.is_child_parent(
child_in_question=child_in_question,
parent_in_question=parent_in_question):
return
except UnknownChildRevision:
# The child_in_question is an unknown SHA. This shouldn't happen.
logging.exception(
"Child SHA is missing from the VCS. This is bad news and "
"shouldn't happen. (parent=%s, child=%s)",
parent_in_question, child_in_question)
return
except UnknownParentRevision:
# The parent_in_question is an unknown SHA. Assume it was
# deleted and forgotten VCS, and set the new green build to
# the child_in_question anyway.
logging.warning(
"Parent SHA is missing from the VCS. Assume it was deleted "
"and the new build is legit.",
extra={
'data': {
'repository_url': source.revision.repository.url,
'branch': branch,
'parent': parent_in_question,
'child': child_in_question,
},
}
)
# switch latest_green_build to this sha
green_build, _ = create_or_update(LatestGreenBuild, where={
'project_id': project.id,
'branch': branch,
}, values={
'build': build,
})
def _set_latest_green_build_for_each_branch(build, source, vcs):
project = build.project
for branch in source.revision.branches:
current_latest_green_build = LatestGreenBuild.query.options(
joinedload('build').joinedload('source')
).filter(
LatestGreenBuild.project_id == project.id,
LatestGreenBuild.branch == branch).first()
if not current_latest_green_build or vcs.is_child_parent(
child_in_question=source.revision_sha,
parent_in_question=current_latest_green_build.build.source.revision_sha):
# switch latest_green_build to this sha
green_build, _ = create_or_update(LatestGreenBuild, where={
'project_id': project.id,
'branch': branch,
}, values={
'build': build,
})
def save(self, repository):
author = self._get_author(self.author)
if self.author == self.committer:
committer = author
else:
committer = self._get_author(self.committer)
revision, created = create_or_update(Revision,
where={
'repository': repository,
'sha': self.id,
},
values={
'author':
author,
'committer':
committer,
'message':
self.message,
'parents':
self.parents,
'branches':
self.branches,
'date_created':
self.author_date,
'date_committed':
self.committer_date,
})
# This call is relatively expensive - only do if necessary.
if created:
vcs = repository.get_vcs()
if vcs:
revision.patch_hash = vcs.get_patch_hash(self.id)
# we also want to create a source for this item as it's the canonical
# representation in the UI
source = try_create(Source, {
'revision_sha': self.id,
'repository': repository,
})
return (revision, created, source)
def _sync_artifact_as_log(self, artifact):
jobstep = artifact.step
job = artifact.job
logsource, created = get_or_create(LogSource,
where={
'name':
artifact.data['displayPath'],
'job': job,
'step': jobstep,
},
defaults={
'job': job,
'project': job.project,
'date_created':
job.date_started,
})
url = '{base}/job/{job}/{build}/artifact/{artifact}'.format(
base=self.base_url,
job=jobstep.data['job_name'],
build=jobstep.data['build_no'],
artifact=artifact.data['relativePath'],
)
offset = 0
session = requests.Session()
with closing(session.get(url, stream=True, timeout=15)) as resp:
iterator = resp.iter_content()
for chunk in chunked(iterator, LOG_CHUNK_SIZE):
chunk_size = len(chunk)
chunk, _ = create_or_update(LogChunk,
where={
'source': logsource,
'offset': offset,
},
values={
'job': job,
'project': job.project,
'size': chunk_size,
'text': chunk,
})
offset += chunk_size
def _set_latest_green_build_for_each_branch(build, source, vcs):
project = build.project
for branch in source.revision.branches:
current_latest_green_build = LatestGreenBuild.query.options(
joinedload('build').joinedload('source')
).filter(
LatestGreenBuild.project_id == project.id,
LatestGreenBuild.branch == branch).first()
if not current_latest_green_build or vcs.is_child_parent(
child_in_question=source.revision_sha,
parent_in_question=current_latest_green_build.build.source.revision_sha):
# switch latest_green_build to this sha
green_build, _ = create_or_update(LatestGreenBuild, where={
'project_id': project.id,
'branch': branch,
}, values={
'build': build,
})
def post(self):
""" HTTP POST to create or update AdminMessages.
This API enforces that we only ever have at most one message. This will
likely change in the future. To clear the current message, post an empty
message. Messages cannot be deleted once created.
Returns:
str: JSON representation of the AdminMessage that was edited.
"""
args = self.post_parser.parse_args()
# Enforce we only ever have a single message
message, _ = create_or_update(AdminMessage, where={}, values={
'message': args.message,
'user_id': get_current_user().id,
'date_created': datetime.utcnow()
})
# Response isn't required, but we give one to make testing easier
return self.respond(message)
def build_finished_handler(build_id, **kwargs):
build = Build.query.get(build_id)
if build is None:
return
if build.cause == Cause.snapshot:
return
if build.result != Result.passed:
return
url = current_app.config.get('GREEN_BUILD_URL')
if not url:
logger.info('GREEN_BUILD_URL not set')
return
auth = current_app.config['GREEN_BUILD_AUTH']
if not auth:
logger.info('GREEN_BUILD_AUTH not set')
return
source = build.source
# we only want to identify stable revisions
if not source.is_commit():
logger.debug('Ignoring build due to non-commit: %s', build.id)
return
options = get_options(build.project_id)
if options.get('green-build.notify', '1') != '1':
logger.info('green-build.notify disabled for project: %s', build.project_id)
return
vcs = source.repository.get_vcs()
if vcs is None:
logger.info('Repository has no VCS set: %s', source.repository.id)
return
# ensure we have the latest changes
if vcs.exists():
vcs.update()
else:
vcs.clone()
release_id = get_release_id(source, vcs)
project = options.get('green-build.project') or build.project.slug
logging.info('Making green_build request to %s', url)
try:
requests.post(url, auth=auth, data={
'project': project,
'id': release_id,
'build_url': build_uri('/projects/{0}/builds/{1}/'.format(
build.project.slug, build.id.hex)),
'build_server': 'changes',
})
except Exception:
logger.exception('Failed to report green build')
status = 'fail'
else:
status = 'success'
create_or_update(Event, where={
'type': EventType.green_build,
'item_id': build.id,
}, values={
'data': {
'status': status,
},
'date_modified': datetime.utcnow(),
})
# set latest_green_build if latest for each branch:
_set_latest_green_build_for_each_branch(build, source, vcs)
