def run_check(command, **kwargs): '''Run a command through subprocess and check for output ''' if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = ' '.join(command) returncode, output, error = run(command, **kwargs) if returncode != 0: secrets = kwargs.get('secrets', []) log.info( 'Command failed with code: {}'.format(returncode), command=hide_secrets(command_as_string, secrets), output=hide_secrets(output, secrets), error=hide_secrets(error, secrets), ) raise click.ClickException( hide_secrets( '`{}` failed with code: {}.'.format( command[0], returncode, ), secrets)) return output
def create_taskcluster_step(uid, body): '''creates taskcluster step''' step = TaskclusterStep() step.uid = uid step.state = 'running' step.task_group_id = body['taskGroupId'] step.scheduler_api = body['schedulerAPI'] log.info('creating taskcluster step %s for task_group_id %s', step.uid, step.task_group_id) db.session.add(step) try: db.session.commit() except IntegrityError as e: # TODO is there a better way to do this? if 'shipit_taskcluster_steps_pkey' in str(e): title = 'Step with that uid already exists' elif 'shipit_taskcluster_steps_task_group_id_key' in str(e): title = 'Step with that task_group_id already exists' else: title = 'Integrity Error' return {'error_title': title, 'error_message': str(e)}, 409 return None
def batch_checkout(repo_url, repo_dir, revision=b"tip", batch_size=100000): """ Helper to clone a mercurial repository using several steps to minimize memory footprint and stay below 1Gb of RAM It's used on Heroku small dynos, and support restarts """ assert isinstance(revision, bytes) assert isinstance(batch_size, int) assert batch_size > 1 log.info("Batch checkout", url=repo_url, dir=repo_dir, size=batch_size) try: cmd = hglib.util.cmdbuilder("clone", repo_url, repo_dir, noupdate=True, verbose=True, stream=True) hg_run(cmd) log.info("Initial clone finished") except hglib.error.CommandError as e: if e.err.startswith("abort: destination '{}' is not empty".format(repo_dir).encode("utf-8")): log.info("Repository already present, skipping clone") else: raise repo = hglib.open(repo_dir) start = max(int(repo.identify(num=True).strip().decode("utf-8")), 1) target = int(repo.identify(rev=revision, num=True).strip().decode("utf-8")) if start >= target: return log.info("Will process checkout in range", start=start, target=target) steps = list(range(start, target, batch_size)) + [target] for rev in steps: log.info("Moving repo to revision", dir=repo_dir, rev=rev) repo.update(rev=rev)
def get_taskcluster_step_status(uid): ''' Get the current status of a taskcluster step ''' log.info('getting step status %s', uid) try: step = db.session.query(TaskclusterStep).filter( TaskclusterStep.uid == uid).one() except NoResultFound: abort(404) if not step.state.value == 'completed': # only poll taskcluster if the step is not resolved successfully # this is so the shipit taskcluster step can still be manually overridden as complete task_group_state = get_taskcluster_tasks_state(step.task_group_id, step.scheduler_api) if step.state.value != task_group_state: # update step status! if task_group_state in ['completed', 'failed', 'exception']: step.finished = datetime.datetime.utcnow() step.state = task_group_state db.session.commit() return dict(uid=step.uid, task_group_id=step.task_group_id, state=step.state.value, finished=step.finished or '', created=step.created)
def create_pipeline(uid, pipeline): log.info('creating pipeline %s', uid) pipeline_steps = [ PipelineStep.from_dict(step) for step in pipeline['parameters']['steps'] ] PIPELINES[uid] = ['running', pipeline_steps]
def run_check(command, **kwargs): """Run a command through subprocess and check for output """ if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = ' ' .join(command) returncode, output, error = run(command, **kwargs) if returncode != 0: log.info( 'Command failed with code: {}'.format(returncode), command=command_as_string, output=output, error=error, ) raise click.ClickException( '`{}` failed with code: {}.'.format( command[0], returncode, ) ) return output
def run_check(command, **kwargs): '''Run a command through subprocess and check for output ''' if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = ' ' .join(command) returncode, output, error = run(command, **kwargs) if returncode != 0: secrets = kwargs.get('secrets', []) log.info( 'Command failed with code: {}'.format(returncode), command=hide_secrets(command_as_string, secrets), output=hide_secrets(output, secrets), error=hide_secrets(error, secrets), ) raise click.ClickException(hide_secrets( '`{}` failed with code: {}.'.format( command[0], returncode, ), secrets) ) return output
def _notify_status_change(trees_changes, tags=[]): if current_app.config.get('PULSE_TREESTATUS_ENABLE'): routing_key_pattern = 'tree/{0}/status_change' exchange = current_app.config.get('PULSE_TREESTATUS_EXCHANGE') for tree_change in trees_changes: tree, status_from, status_to = tree_change payload = {'status_from': status_from, 'status_to': status_to, 'tree': tree.to_dict(), 'tags': tags} routing_key = routing_key_pattern.format(tree.tree) log.info( 'Sending pulse to {} for tree: {}'.format( exchange, tree.tree, )) try: current_app.pulse.publish(exchange, routing_key, payload) except Exception as e: import traceback msg = 'Can\'t send notification to pulse.' trace = traceback.format_exc() log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def revoke_token(id): '''Revoke an authentication token, identified by its ID. The caller must have permission to revoke this type of token; if that is a ``.my`` permission, then the user email must match. The response status is 204 on success. Revoking an already-revoked token returns 403.''' session = flask.current_app.db.session token_data = tokens_api.models.Token.query.filter_by(id=id).first() # don't leak info about which tokens exist -- return the same # status whether the token is missing or permission is missing if not token_data: raise werkzeug.exceptions.Forbidden if not can_access_token('revoke', token_data.typ): raise werkzeug.exceptions.Forbidden perms_str = ', '.join(str(p) for p in token_data.permissions) log = logger.bind(token_typ=token_data.typ, token_permissions=perms_str, token_id=id, mozdef=True) log.info('Revoking {} token #{} with permissions {}'.format( token_data.typ, token_data.id, perms_str)) tokens_api.models.Token.query.filter_by(id=id).delete() session.commit() return None, 204
def _notify_status_change(trees_changes, tags=[]): if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'): routing_key_pattern = 'tree/{0}/status_change' exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE') for tree_change in trees_changes: tree, status_from, status_to = tree_change payload = {'status_from': status_from, 'status_to': status_to, 'tree': tree.to_dict(), 'tags': tags} routing_key = routing_key_pattern.format(tree.tree) log.info( 'Sending pulse to {} for tree: {}'.format( exchange, tree.tree, )) try: flask.current_app.pulse.publish(exchange, routing_key, payload) except Exception as e: import traceback msg = 'Can\'t send notification to pulse.' trace = traceback.format_exc() log.error('{0}\nException:{1}\nTraceback: {2}'.format(msg, e, trace)) # noqa
def write_user_config(self, config): ''' Update local user preferences ''' user_config = deep_merge(self.load_file(self.user_config_path), config) with open(self.user_config_path, 'w') as f: toml.dump(user_config, f) log.info('Updated local config', path=self.user_config_path)
def _log_process(output, name): # Read and display every line out = output.read() if out is None: return text = filter(None, out.decode('utf-8').splitlines()) for line in text: log.info('{}: {}'.format(name, line))
def write_user_config(self, config): ''' Update local user preferences ''' user_config = deep_merge(self.load_file(self.user_config_path), config) with open(self.user_config_path, 'w') as f: toml.dump(user_config, f) log.info('Updated local config', path=self.user_config_path)
def _log_process(output, name): # Read and display every line out = output.read() if out is None: return text = filter(None, out.decode('utf-8').splitlines()) for line in text: log.info('{}: {}'.format(name, line))
def _statuspage_resolve_incident( headers, component_id, tree, status_from, status_to, ): page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID') response = requests.get( f'{STATUSPAGE_URL}/pages/{page_id}/incidents/unresolved', headers=headers, ) try: response.raise_for_status() except Exception as e: log.exception(e) _statuspage_send_email_on_error( subject=f'[treestatus] Error when closing statuspage incident', content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree), ) return # last incident with meta.treestatus.tree == tree.tree incident_id = None incidents = sorted(response.json(), key=lambda x: x['created_at']) for incident in incidents: if 'id' in incident and \ 'metadata' in incident and \ 'treestatus' in incident['metadata'] and \ 'tree' in incident['metadata']['treestatus'] and \ incident['metadata']['treestatus']['tree'] == tree.tree: incident_id = incident['id'] break if incident_id is None: log.info(f'No incident found when closing tree `{tree.tree}`') return response = requests.patch( f'{STATUSPAGE_URL}/pages/{page_id}/incidents/{incident_id}', headers=headers, json=_statuspage_data( True, component_id, tree, status_from, status_to, ), ) try: response.raise_for_status() except Exception as e: log.exception(e) _statuspage_send_email_on_error( subject=f'[treestatus] Error when closing statuspage incident', content=STATUSPAGE_ERROR_ON_CLOSE.format(tree=tree.tree), incident_id=incident_id, )
def load_file(self, path): ''' Load a TOML config from a local file ''' if not os.path.exists(path): log.info('Missing local config', path=path) return {} data = toml.load(open(path)) log.info('Read local config', path=path) return data
def load_file(self, path): ''' Load a TOML config from a local file ''' if not os.path.exists(path): log.info('Missing local config', path=path) return {} data = toml.load(open(path)) log.info('Read local config', path=path) return data
def issue_token(body): '''Issue a new token. The body should not include a ``token`` or ``id``, but should include a ``typ`` and the necessary fields for that type. The response will contain both ``token`` and ``id``. You must have permission to issue the given token type.''' typ = body['typ'] # verify permission to issue this type permission = '{}/{}/issue'.format(releng_tokens.config.SCOPE_PREFIX, typ) if not flask_login.current_user.has_permissions(permission): raise werkzeug.exceptions.Forbidden( 'You do not have permission to create this token type') # verify required parameters; any extras will be ignored for attr in required_token_attributes[typ]: if body.get(attr, UNSET) is UNSET: raise werkzeug.exceptions.BadRequest('missing %s' % attr) # prohibit silly requests if body.get('disabled'): raise werkzeug.exceptions.BadRequest('can\'t issue disabled tokens') # All types have permissions, so handle those here -- ensure the request is # for a subset of the permissions the user can perform all_relengapi_permissions = [ (i, backend_common.auth.from_relengapi_permission(i)) for i in backend_common.auth.RELENGAPI_PERMISSIONS.keys() ] requested_permissions = [ old for old, new in all_relengapi_permissions if flask_login.current_user.has_permissions(new) and old in body.get('permissions', []) ] if None in requested_permissions: raise werkzeug.exceptions.BadRequest('bad permissions') if not set(requested_permissions) <= set( [i for i, j in all_relengapi_permissions]): raise werkzeug.exceptions.BadRequest('bad permissions') # Dispatch the rest to the per-type function. Note that WSME has already # ensured `typ` is one of the recognized types. token = token_issuers[typ](body, requested_permissions) perms_str = ', '.join(str(p) for p in requested_permissions) log = logger.bind(token_typ=token['typ'], token_permissions=perms_str, mozdef=True) if token.get('id'): log = log.bind(token_id=token['id']) log.info('Issuing {} token to {} with permissions {}'.format( token['typ'], flask_login.current_user, perms_str)) return dict(result=token)
def get_taskcluster_step(uid): log.info('getting step %s', uid) try: step = db.session.query(TaskclusterStep).filter( TaskclusterStep.uid == uid).one() except NoResultFound: abort(404, 'taskcluster step not found') return dict(uid=step.uid, taskGroupId=step.task_group_id, schedulerAPI=step.scheduler_api, parameters={})
def batch_checkout(repo_url, repo_dir, revision=b'tip', batch_size=100000): ''' Helper to clone a mercurial repository using several steps to minimize memory footprint and stay below 1Gb of RAM It's used on Heroku small dynos ''' assert isinstance(revision, bytes) assert isinstance(batch_size, int) assert batch_size > 1 log.info('Batch checkout', url=repo_url, dir=repo_dir, size=batch_size) cmd = hglib.util.cmdbuilder('clone', repo_url, repo_dir, noupdate=True, stream=True) hg_run(cmd) log.info('Inital clone finished') repo = hglib.open(repo_dir) target = int(repo.identify(rev=revision, num=True).strip().decode('utf-8')) log.info('Target revision for incremental checkout', revision=target) steps = list(range(1, target, batch_size)) + [target] for rev in steps: log.info('Moving repo to revision', dir=repo_dir, rev=rev) repo.update(rev=rev) return repo
def send_notifications(message: Message, identity_preference: dict) -> dict: try: response = CHANNEL_MAPPING[identity_preference['channel']]( message, identity_preference) log.info('{target} notified about {message} on {channel}'.format( target=identity_preference['target'], message=message, channel=identity_preference['channel'])) return response except TaskclusterFailure: pass
def issue_token(body): '''Issue a new token. The body should not include a ``token`` or ``id``, but should include a ``typ`` and the necessary fields for that type. The response will contain both ``token`` and ``id``. You must have permission to issue the given token type.''' typ = body['typ'] # verify permission to issue this type permission = '{}/{}/issue'.format(tokens_api.config.SCOPE_PREFIX, typ) if not flask_login.current_user.has_permissions(permission): raise werkzeug.exceptions.Forbidden( 'You do not have permission to create this token type') # verify required parameters; any extras will be ignored for attr in required_token_attributes[typ]: if body.get(attr, UNSET) is UNSET: raise werkzeug.exceptions.BadRequest('missing %s' % attr) # prohibit silly requests if body.get('disabled'): raise werkzeug.exceptions.BadRequest('can\'t issue disabled tokens') # All types have permissions, so handle those here -- ensure the request is # for a subset of the permissions the user can perform all_relengapi_permissions = [ (i, backend_common.auth.from_relengapi_permission(i)) for i in backend_common.auth.RELENGAPI_PERMISSIONS.keys() ] requested_permissions = [ old for old, new in all_relengapi_permissions if flask_login.current_user.has_permissions(new) and old in body.get('permissions', []) ] if None in requested_permissions: raise werkzeug.exceptions.BadRequest('bad permissions') if not set(requested_permissions) <= set([i for i, j in all_relengapi_permissions]): raise werkzeug.exceptions.BadRequest('bad permissions') # Dispatch the rest to the per-type function. Note that WSME has already # ensured `typ` is one of the recognized types. token = token_issuers[typ](body, requested_permissions) perms_str = ', '.join(str(p) for p in requested_permissions) log = logger.bind(token_typ=token['typ'], token_permissions=perms_str, mozdef=True) if token.get('id'): log = log.bind(token_id=token['id']) log.info('Issuing {} token to {} with permissions {}'.format( token['typ'], flask_login.current_user, perms_str)) return dict(result=token)
def delete_taskcluster_step(uid): log.info('deleting step %s', uid) try: step = TaskclusterStep.query.filter_by(uid=uid).one() except Exception as e: exception = 'Taskcluster step could not be found by given uid: {}'.format( uid) log.exception(exception) abort(404, exception) db.session.delete(step) db.session.commit() return {}
def list_taskcluster_steps(state='running'): log.info('listing steps') try: desired_state = TaskclusterStatus[state] except KeyError: exception = '{} is not a valid state'.format(state) log.warning('valid states: %s', [state.value for state in TaskclusterStatus]) log.exception(exception) abort(400, exception) try: steps = db.session.query(TaskclusterStep).filter( TaskclusterStep.state == desired_state).all() except NoResultFound: abort(404, 'No Taskcluster steps found with that given state.') log.info('listing steps: {}', steps) return [step.uid for step in steps]
def run_check(command, **kwargs): """Run a command through subprocess and check for output """ if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = " ".join(command) returncode, output, error = run(command, **kwargs) if returncode != 0: secrets = kwargs.get("secrets", []) log.info( f"Command failed with code: {returncode}", command=hide_secrets(command_as_string, secrets), output=hide_secrets(output, secrets), error=hide_secrets(error, secrets), ) raise click.ClickException(hide_secrets(f"`{command[0]}` failed with code: {returncode}.", secrets)) return output
def batch_checkout(repo_url, repo_dir, revision=b'tip', batch_size=100000): ''' Helper to clone a mercurial repository using several steps to minimize memory footprint and stay below 1Gb of RAM It's used on Heroku small dynos, and support restarts ''' assert isinstance(revision, bytes) assert isinstance(batch_size, int) assert batch_size > 1 log.info('Batch checkout', url=repo_url, dir=repo_dir, size=batch_size) try: cmd = hglib.util.cmdbuilder('clone', repo_url, repo_dir, noupdate=True, verbose=True, stream=True) hg_run(cmd) log.info('Initial clone finished') except hglib.error.CommandError as e: if e.err.startswith('abort: destination \'{}\' is not empty'.format(repo_dir).encode('utf-8')): log.info('Repository already present, skipping clone') else: raise repo = hglib.open(repo_dir) start = max(int(repo.identify(num=True).strip().decode('utf-8')), 1) target = int(repo.identify(rev=revision, num=True).strip().decode('utf-8')) if start >= target: return log.info('Will process checkout in range', start=start, target=target) steps = list(range(start, target, batch_size)) + [target] for rev in steps: log.info('Moving repo to revision', dir=repo_dir, rev=rev) repo.update(rev=rev)
def get_pipeline_status(uid): log.info('getting pipeline status %s', uid) return dict(state=PIPELINES[uid][0])
def list_pipelines(): log.info('listing pipelines') return list(PIPELINES.keys())
def get_pipeline(uid): log.info('getting pipeline %s', uid) try: return PIPELINES[uid][1] except (KeyError, ValueError): return None, 404
def delete_pipeline(uid): log.info('deleting pipeline %s', uid) del PIPELINES[uid]
def _notify_status_change(trees_changes, tags=[]): if flask.current_app.config.get('STATUSPAGE_ENABLE'): log.debug('Notify statuspage about trees changes.') components = flask.current_app.config.get('STATUSPAGE_COMPONENTS', {}) token = flask.current_app.config.get('STATUSPAGE_TOKEN') if not token: log.error('STATUSPAGE_PAGE_ID not defined in app config.') else: headers = {'Authorization': f'OAuth {token}'} for tree_change in trees_changes: tree, status_from, status_to = tree_change if tree.tree not in components.keys(): continue log.debug(f'Notify statuspage about: {tree.tree}') component_id = components[tree.tree] # create an accident if status_from in ['open', 'approval required' ] and status_to == 'closed': _statuspage_create_incident( headers, component_id, tree, status_from, status_to, ) # close an accident elif status_from == 'closed' and status_to in [ 'open', 'approval required' ]: _statuspage_resolve_incident( headers, component_id, tree, status_from, status_to, ) if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'): routing_key_pattern = 'tree/{0}/status_change' exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE') for tree_change in trees_changes: tree, status_from, status_to = tree_change payload = { 'status_from': status_from, 'status_to': status_to, 'tree': tree.to_dict(), 'tags': tags } routing_key = routing_key_pattern.format(tree.tree) log.info('Sending pulse to {} for tree: {}'.format( exchange, tree.tree, )) try: flask.current_app.pulse.publish(exchange, routing_key, payload) except Exception as e: import traceback msg = 'Can\'t send notification to pulse.' trace = traceback.format_exc() log.error('{0}\nException:{1}\nTraceback: {2}'.format( msg, e, trace)) # noqa