def run_check(command, **kwargs):
'''Run a command through subprocess and check for output
'''
if type(command) is str:
command_as_string = command
command = shlex.split(command)
else:
command_as_string = ' '.join(command)
returncode, output, error = run(command, **kwargs)
if returncode != 0:
secrets = kwargs.get('secrets', [])
log.info(
'Command failed with code: {}'.format(returncode),
command=hide_secrets(command_as_string, secrets),
output=hide_secrets(output, secrets),
error=hide_secrets(error, secrets),
)
raise click.ClickException(
hide_secrets(
'`{}` failed with code: {}.'.format(
command[0],
returncode,
), secrets))
return output
def create_taskcluster_step(uid, body):
'''creates taskcluster step'''
step = TaskclusterStep()
step.uid = uid
step.state = 'running'
step.task_group_id = body['taskGroupId']
step.scheduler_api = body['schedulerAPI']
log.info('creating taskcluster step %s for task_group_id %s', step.uid,
step.task_group_id)
db.session.add(step)
try:
db.session.commit()
except IntegrityError as e:
# TODO is there a better way to do this?
if 'shipit_taskcluster_steps_pkey' in str(e):
title = 'Step with that uid already exists'
elif 'shipit_taskcluster_steps_task_group_id_key' in str(e):
title = 'Step with that task_group_id already exists'
else:
title = 'Integrity Error'
return {'error_title': title, 'error_message': str(e)}, 409
return None
def batch_checkout(repo_url, repo_dir, revision=b"tip", batch_size=100000):
"""
Helper to clone a mercurial repository using several steps
to minimize memory footprint and stay below 1Gb of RAM
It's used on Heroku small dynos, and support restarts
"""
assert isinstance(revision, bytes)
assert isinstance(batch_size, int)
assert batch_size > 1
log.info("Batch checkout", url=repo_url, dir=repo_dir, size=batch_size)
try:
cmd = hglib.util.cmdbuilder("clone", repo_url, repo_dir, noupdate=True, verbose=True, stream=True)
hg_run(cmd)
log.info("Initial clone finished")
except hglib.error.CommandError as e:
if e.err.startswith("abort: destination '{}' is not empty".format(repo_dir).encode("utf-8")):
log.info("Repository already present, skipping clone")
else:
raise
repo = hglib.open(repo_dir)
start = max(int(repo.identify(num=True).strip().decode("utf-8")), 1)
target = int(repo.identify(rev=revision, num=True).strip().decode("utf-8"))
if start >= target:
return
log.info("Will process checkout in range", start=start, target=target)
steps = list(range(start, target, batch_size)) + [target]
for rev in steps:
log.info("Moving repo to revision", dir=repo_dir, rev=rev)
repo.update(rev=rev)
def get_taskcluster_step_status(uid):
'''
Get the current status of a taskcluster step
'''
log.info('getting step status %s', uid)
try:
step = db.session.query(TaskclusterStep).filter(
TaskclusterStep.uid == uid).one()
except NoResultFound:
abort(404)
if not step.state.value == 'completed':
# only poll taskcluster if the step is not resolved successfully
# this is so the shipit taskcluster step can still be manually overridden as complete
task_group_state = get_taskcluster_tasks_state(step.task_group_id,
step.scheduler_api)
if step.state.value != task_group_state:
# update step status!
if task_group_state in ['completed', 'failed', 'exception']:
step.finished = datetime.datetime.utcnow()
step.state = task_group_state
db.session.commit()
return dict(uid=step.uid,
task_group_id=step.task_group_id,
state=step.state.value,
finished=step.finished or '',
created=step.created)
def create_pipeline(uid, pipeline):
log.info('creating pipeline %s', uid)
pipeline_steps = [
PipelineStep.from_dict(step)
for step in pipeline['parameters']['steps']
]
PIPELINES[uid] = ['running', pipeline_steps]
def run_check(command, **kwargs):
"""Run a command through subprocess and check for output
"""
if type(command) is str:
command_as_string = command
command = shlex.split(command)
else:
command_as_string = ' ' .join(command)
returncode, output, error = run(command, **kwargs)
if returncode != 0:
log.info(
'Command failed with code: {}'.format(returncode),
command=command_as_string,
output=output,
error=error,
)
raise click.ClickException(
'`{}` failed with code: {}.'.format(
command[0],
returncode,
)
)
return output
def run_check(command, **kwargs):
'''Run a command through subprocess and check for output
'''
if type(command) is str:
command_as_string = command
command = shlex.split(command)
else:
command_as_string = ' ' .join(command)
returncode, output, error = run(command, **kwargs)
if returncode != 0:
secrets = kwargs.get('secrets', [])
log.info(
'Command failed with code: {}'.format(returncode),
command=hide_secrets(command_as_string, secrets),
output=hide_secrets(output, secrets),
error=hide_secrets(error, secrets),
)
raise click.ClickException(hide_secrets(
'`{}` failed with code: {}.'.format(
command[0],
returncode,
), secrets)
)
return output
def _notify_status_change(trees_changes, tags=[]):
if current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags}
routing_key = routing_key_pattern.format(tree.tree)
log.info(
'Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def revoke_token(id):
'''Revoke an authentication token, identified by its ID.
The caller must have permission to revoke this type of token; if
that is a ``.my`` permission, then the user email must match.
The response status is 204 on success. Revoking an already-revoked token
returns 403.'''
session = flask.current_app.db.session
token_data = tokens_api.models.Token.query.filter_by(id=id).first()
# don't leak info about which tokens exist -- return the same
# status whether the token is missing or permission is missing
if not token_data:
raise werkzeug.exceptions.Forbidden
if not can_access_token('revoke', token_data.typ):
raise werkzeug.exceptions.Forbidden
perms_str = ', '.join(str(p) for p in token_data.permissions)
log = logger.bind(token_typ=token_data.typ, token_permissions=perms_str,
token_id=id, mozdef=True)
log.info('Revoking {} token #{} with permissions {}'.format(
token_data.typ, token_data.id, perms_str))
tokens_api.models.Token.query.filter_by(id=id).delete()
session.commit()
return None, 204
def _notify_status_change(trees_changes, tags=[]):
if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags}
routing_key = routing_key_pattern.format(tree.tree)
log.info(
'Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
flask.current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def write_user_config(self, config):
'''
Update local user preferences
'''
user_config = deep_merge(self.load_file(self.user_config_path), config)
with open(self.user_config_path, 'w') as f:
toml.dump(user_config, f)
log.info('Updated local config', path=self.user_config_path)
def _log_process(output, name):
# Read and display every line
out = output.read()
if out is None:
return
text = filter(None, out.decode('utf-8').splitlines())
for line in text:
log.info('{}: {}'.format(name, line))
def write_user_config(self, config):
'''
Update local user preferences
'''
user_config = deep_merge(self.load_file(self.user_config_path), config)
with open(self.user_config_path, 'w') as f:
toml.dump(user_config, f)
log.info('Updated local config', path=self.user_config_path)
def _log_process(output, name):
# Read and display every line
out = output.read()
if out is None:
return
text = filter(None, out.decode('utf-8').splitlines())
for line in text:
log.info('{}: {}'.format(name, line))
def _statuspage_resolve_incident(
headers,
component_id,
tree,
status_from,
status_to,
):
page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID')
response = requests.get(
f'{STATUSPAGE_URL}/pages/{page_id}/incidents/unresolved',
headers=headers,
)
try:
response.raise_for_status()
except Exception as e:
log.exception(e)
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when closing statuspage incident',
content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree),
)
return
# last incident with meta.treestatus.tree == tree.tree
incident_id = None
incidents = sorted(response.json(), key=lambda x: x['created_at'])
for incident in incidents:
if 'id' in incident and \
'metadata' in incident and \
'treestatus' in incident['metadata'] and \
'tree' in incident['metadata']['treestatus'] and \
incident['metadata']['treestatus']['tree'] == tree.tree:
incident_id = incident['id']
break
if incident_id is None:
log.info(f'No incident found when closing tree `{tree.tree}`')
return
response = requests.patch(
f'{STATUSPAGE_URL}/pages/{page_id}/incidents/{incident_id}',
headers=headers,
json=_statuspage_data(
True,
component_id,
tree,
status_from,
status_to,
),
)
try:
response.raise_for_status()
except Exception as e:
log.exception(e)
_statuspage_send_email_on_error(
subject=f'[treestatus] Error when closing statuspage incident',
content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree),
incident_id=incident_id,
)
def load_file(self, path):
'''
Load a TOML config from a local file
'''
if not os.path.exists(path):
log.info('Missing local config', path=path)
return {}
data = toml.load(open(path))
log.info('Read local config', path=path)
return data
def load_file(self, path):
'''
Load a TOML config from a local file
'''
if not os.path.exists(path):
log.info('Missing local config', path=path)
return {}
data = toml.load(open(path))
log.info('Read local config', path=path)
return data
def issue_token(body):
'''Issue a new token. The body should not include a ``token`` or ``id``,
but should include a ``typ`` and the necessary fields for that type. The
response will contain both ``token`` and ``id``. You must have permission
to issue the given token type.'''
typ = body['typ']
# verify permission to issue this type
permission = '{}/{}/issue'.format(releng_tokens.config.SCOPE_PREFIX, typ)
if not flask_login.current_user.has_permissions(permission):
raise werkzeug.exceptions.Forbidden(
'You do not have permission to create this token type')
# verify required parameters; any extras will be ignored
for attr in required_token_attributes[typ]:
if body.get(attr, UNSET) is UNSET:
raise werkzeug.exceptions.BadRequest('missing %s' % attr)
# prohibit silly requests
if body.get('disabled'):
raise werkzeug.exceptions.BadRequest('can\'t issue disabled tokens')
# All types have permissions, so handle those here -- ensure the request is
# for a subset of the permissions the user can perform
all_relengapi_permissions = [
(i, backend_common.auth.from_relengapi_permission(i))
for i in backend_common.auth.RELENGAPI_PERMISSIONS.keys()
]
requested_permissions = [
old for old, new in all_relengapi_permissions
if flask_login.current_user.has_permissions(new)
and old in body.get('permissions', [])
]
if None in requested_permissions:
raise werkzeug.exceptions.BadRequest('bad permissions')
if not set(requested_permissions) <= set(
[i for i, j in all_relengapi_permissions]):
raise werkzeug.exceptions.BadRequest('bad permissions')
# Dispatch the rest to the per-type function. Note that WSME has already
# ensured `typ` is one of the recognized types.
token = token_issuers[typ](body, requested_permissions)
perms_str = ', '.join(str(p) for p in requested_permissions)
log = logger.bind(token_typ=token['typ'],
token_permissions=perms_str,
mozdef=True)
if token.get('id'):
log = log.bind(token_id=token['id'])
log.info('Issuing {} token to {} with permissions {}'.format(
token['typ'], flask_login.current_user, perms_str))
return dict(result=token)
def get_taskcluster_step(uid):
log.info('getting step %s', uid)
try:
step = db.session.query(TaskclusterStep).filter(
TaskclusterStep.uid == uid).one()
except NoResultFound:
abort(404, 'taskcluster step not found')
return dict(uid=step.uid,
taskGroupId=step.task_group_id,
schedulerAPI=step.scheduler_api,
parameters={})
def batch_checkout(repo_url, repo_dir, revision=b'tip', batch_size=100000):
'''
Helper to clone a mercurial repository using several steps
to minimize memory footprint and stay below 1Gb of RAM
It's used on Heroku small dynos
'''
assert isinstance(revision, bytes)
assert isinstance(batch_size, int)
assert batch_size > 1
log.info('Batch checkout', url=repo_url, dir=repo_dir, size=batch_size)
cmd = hglib.util.cmdbuilder('clone',
repo_url,
repo_dir,
noupdate=True,
stream=True)
hg_run(cmd)
log.info('Inital clone finished')
repo = hglib.open(repo_dir)
target = int(repo.identify(rev=revision, num=True).strip().decode('utf-8'))
log.info('Target revision for incremental checkout', revision=target)
steps = list(range(1, target, batch_size)) + [target]
for rev in steps:
log.info('Moving repo to revision', dir=repo_dir, rev=rev)
repo.update(rev=rev)
return repo
def send_notifications(message: Message, identity_preference: dict) -> dict:
try:
response = CHANNEL_MAPPING[identity_preference['channel']](
message, identity_preference)
log.info('{target} notified about {message} on {channel}'.format(
target=identity_preference['target'],
message=message,
channel=identity_preference['channel']))
return response
except TaskclusterFailure:
pass
def issue_token(body):
'''Issue a new token. The body should not include a ``token`` or ``id``,
but should include a ``typ`` and the necessary fields for that type. The
response will contain both ``token`` and ``id``. You must have permission
to issue the given token type.'''
typ = body['typ']
# verify permission to issue this type
permission = '{}/{}/issue'.format(tokens_api.config.SCOPE_PREFIX, typ)
if not flask_login.current_user.has_permissions(permission):
raise werkzeug.exceptions.Forbidden(
'You do not have permission to create this token type')
# verify required parameters; any extras will be ignored
for attr in required_token_attributes[typ]:
if body.get(attr, UNSET) is UNSET:
raise werkzeug.exceptions.BadRequest('missing %s' % attr)
# prohibit silly requests
if body.get('disabled'):
raise werkzeug.exceptions.BadRequest('can\'t issue disabled tokens')
# All types have permissions, so handle those here -- ensure the request is
# for a subset of the permissions the user can perform
all_relengapi_permissions = [
(i, backend_common.auth.from_relengapi_permission(i))
for i in backend_common.auth.RELENGAPI_PERMISSIONS.keys()
]
requested_permissions = [
old
for old, new in all_relengapi_permissions
if flask_login.current_user.has_permissions(new) and old in body.get('permissions', [])
]
if None in requested_permissions:
raise werkzeug.exceptions.BadRequest('bad permissions')
if not set(requested_permissions) <= set([i for i, j in all_relengapi_permissions]):
raise werkzeug.exceptions.BadRequest('bad permissions')
# Dispatch the rest to the per-type function. Note that WSME has already
# ensured `typ` is one of the recognized types.
token = token_issuers[typ](body, requested_permissions)
perms_str = ', '.join(str(p) for p in requested_permissions)
log = logger.bind(token_typ=token['typ'], token_permissions=perms_str, mozdef=True)
if token.get('id'):
log = log.bind(token_id=token['id'])
log.info('Issuing {} token to {} with permissions {}'.format(
token['typ'], flask_login.current_user, perms_str))
return dict(result=token)
def delete_taskcluster_step(uid):
log.info('deleting step %s', uid)
try:
step = TaskclusterStep.query.filter_by(uid=uid).one()
except Exception as e:
exception = 'Taskcluster step could not be found by given uid: {}'.format(
uid)
log.exception(exception)
abort(404, exception)
db.session.delete(step)
db.session.commit()
return {}
def list_taskcluster_steps(state='running'):
log.info('listing steps')
try:
desired_state = TaskclusterStatus[state]
except KeyError:
exception = '{} is not a valid state'.format(state)
log.warning('valid states: %s',
[state.value for state in TaskclusterStatus])
log.exception(exception)
abort(400, exception)
try:
steps = db.session.query(TaskclusterStep).filter(
TaskclusterStep.state == desired_state).all()
except NoResultFound:
abort(404, 'No Taskcluster steps found with that given state.')
log.info('listing steps: {}', steps)
return [step.uid for step in steps]
def run_check(command, **kwargs):
"""Run a command through subprocess and check for output
"""
if type(command) is str:
command_as_string = command
command = shlex.split(command)
else:
command_as_string = " ".join(command)
returncode, output, error = run(command, **kwargs)
if returncode != 0:
secrets = kwargs.get("secrets", [])
log.info(
f"Command failed with code: {returncode}",
command=hide_secrets(command_as_string, secrets),
output=hide_secrets(output, secrets),
error=hide_secrets(error, secrets),
)
raise click.ClickException(hide_secrets(f"`{command[0]}` failed with code: {returncode}.", secrets))
return output
def batch_checkout(repo_url, repo_dir, revision=b'tip', batch_size=100000):
'''
Helper to clone a mercurial repository using several steps
to minimize memory footprint and stay below 1Gb of RAM
It's used on Heroku small dynos, and support restarts
'''
assert isinstance(revision, bytes)
assert isinstance(batch_size, int)
assert batch_size > 1
log.info('Batch checkout', url=repo_url, dir=repo_dir, size=batch_size)
try:
cmd = hglib.util.cmdbuilder('clone',
repo_url,
repo_dir,
noupdate=True,
verbose=True,
stream=True)
hg_run(cmd)
log.info('Initial clone finished')
except hglib.error.CommandError as e:
if e.err.startswith('abort: destination \'{}\' is not empty'.format(repo_dir).encode('utf-8')):
log.info('Repository already present, skipping clone')
else:
raise
repo = hglib.open(repo_dir)
start = max(int(repo.identify(num=True).strip().decode('utf-8')), 1)
target = int(repo.identify(rev=revision, num=True).strip().decode('utf-8'))
if start >= target:
return
log.info('Will process checkout in range', start=start, target=target)
steps = list(range(start, target, batch_size)) + [target]
for rev in steps:
log.info('Moving repo to revision', dir=repo_dir, rev=rev)
repo.update(rev=rev)
def get_pipeline_status(uid):
log.info('getting pipeline status %s', uid)
return dict(state=PIPELINES[uid][0])
def list_pipelines():
log.info('listing pipelines')
return list(PIPELINES.keys())
def get_pipeline(uid):
log.info('getting pipeline %s', uid)
try:
return PIPELINES[uid][1]
except (KeyError, ValueError):
return None, 404
def delete_pipeline(uid):
log.info('deleting pipeline %s', uid)
del PIPELINES[uid]
def _notify_status_change(trees_changes, tags=[]):
if flask.current_app.config.get('STATUSPAGE_ENABLE'):
log.debug('Notify statuspage about trees changes.')
components = flask.current_app.config.get('STATUSPAGE_COMPONENTS', {})
token = flask.current_app.config.get('STATUSPAGE_TOKEN')
if not token:
log.error('STATUSPAGE_PAGE_ID not defined in app config.')
else:
headers = {'Authorization': f'OAuth {token}'}
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
if tree.tree not in components.keys():
continue
log.debug(f'Notify statuspage about: {tree.tree}')
component_id = components[tree.tree]
# create an accident
if status_from in ['open', 'approval required'
] and status_to == 'closed':
_statuspage_create_incident(
headers,
component_id,
tree,
status_from,
status_to,
)
# close an accident
elif status_from == 'closed' and status_to in [
'open', 'approval required'
]:
_statuspage_resolve_incident(
headers,
component_id,
tree,
status_from,
status_to,
)
if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'):
routing_key_pattern = 'tree/{0}/status_change'
exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE')
for tree_change in trees_changes:
tree, status_from, status_to = tree_change
payload = {
'status_from': status_from,
'status_to': status_to,
'tree': tree.to_dict(),
'tags': tags
}
routing_key = routing_key_pattern.format(tree.tree)
log.info('Sending pulse to {} for tree: {}'.format(
exchange,
tree.tree,
))
try:
flask.current_app.pulse.publish(exchange, routing_key, payload)
except Exception as e:
import traceback
msg = 'Can\'t send notification to pulse.'
trace = traceback.format_exc()
log.error('{0}\nException:{1}\nTraceback: {2}'.format(
msg, e, trace)) # noqa
