def _statuspage_create_incident( headers, component_id, tree, status_from, status_to, ): page_id = flask.current_app.config.get('STATUSPAGE_PAGE_ID') if not page_id: log.error('STATUSPAGE_PAGE_ID not defined in app config.') return data = _statuspage_data( False, component_id, tree, status_from, status_to, ) log.debug( f'Create statuspage incident for tree `{tree.tree}` under page `{page_id}`', data=data) response = requests.post( f'{STATUSPAGE_URL}/pages/{page_id}/incidents', headers=headers, json=data, ) try: response.raise_for_status() except Exception as e: log.exception(e) _statuspage_send_email_on_error( subject=f'[treestatus] Error when creating statuspage incident', content=STATUSPAGE_ERROR_ON_CREATE.format(tree=tree.tree), )
def cmd(project, zsh, quiet, command, nix_shell): run = [] if zsh or command: run.append('--run') if command: run.append(command + '; exit') elif zsh: run.append('zsh') _command = [ nix_shell, os.path.join(please_cli.config.ROOT_DIR, 'nix/default.nix'), '-A', project, '-j', '1', ] + run os.environ['SERVICES_ROOT'] = please_cli.config.ROOT_DIR + '/' os.environ['SSL_DEV_CA'] = os.path.join(please_cli.config.TMP_DIR, 'certs') if command: handle_stream_line = None if quiet is False: handle_stream_line = click.echo return cli_common.command.run( _command, stream=True, handle_stream_line=handle_stream_line, stderr=subprocess.STDOUT, ) else: log.debug('Running command using os.system', command=_command) return os.system('PYTHONPATH="" ' + ' '.join(_command)) / 256, '', ''
def run(command, stream=False, handle_stream_line=None, log_command=True, log_output=True, secrets=[], **kwargs): '''Run a command through subprocess ''' if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = ' '.join(command) if len(command) == 0: raise click.ClickException('Can\'t run an empty command.') _kwargs = dict( stdin=subprocess.DEVNULL, # no interactions stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) _kwargs.update(kwargs) if stream: _kwargs['bufsize'] = 1 if log_command: log.debug('Running command', command=hide_secrets(command_as_string, secrets), kwargs=_kwargs) with subprocess.Popen(command, **_kwargs) as proc: if stream: output = [] for line in proc.stdout: line = line.decode('utf-8', 'ignore') line = line.rstrip('\n') if log_output: log.debug(hide_secrets(line, secrets)) output.append(line) if handle_stream_line: handle_stream_line(line) output = '\n'.join(output) # TODO: When needed we should also add possibility to stream stdout # and sterr separatly using asyncio.subprocess: # https://kevinmccarthy.org/2016/07/25/streaming-subprocess-stdin-and-stdout-with-asyncio-in-python/ # You can still pipe stderr into stdout which is enough for now. error = '' else: output, error = proc.communicate() return proc.returncode, output, error
def cmd( project, zsh, quiet, command, nix_shell, taskcluster_secrets, taskcluster_client_id, taskcluster_access_token, ): run = [] if zsh or command: run.append('--run') if command: run.append(command + '; exit') elif zsh: run.append('zsh') _command = [ nix_shell, os.path.join(please_cli.config.ROOT_DIR, 'nix/default.nix'), '-A', project, '-j', '1', ] + run os.environ['SERVICES_ROOT'] = please_cli.config.ROOT_DIR + '/' os.environ['SSL_DEV_CA'] = os.path.join(please_cli.config.TMP_DIR, 'certs') os.environ['PYTHONPATH'] = "" os.environ['TASKCLUSTER_SECRET'] = taskcluster_secrets if taskcluster_client_id: os.environ['TASKCLUSTER_CLIENT_ID'] = taskcluster_client_id if taskcluster_access_token: os.environ['TASKCLUSTER_ACCESS_TOKEN'] = taskcluster_access_token if command: handle_stream_line = None if quiet is False: handle_stream_line = click.echo return cli_common.command.run( _command, stream=True, handle_stream_line=handle_stream_line, stderr=subprocess.STDOUT, ) else: log.debug('Running command using os.system', command=_command) return os.system(' '.join(_command)) / 256, '', ''
def run(command, stream=False, handle_stream_line=None, log_command=True, log_output=True, secrets=[], **kwargs): '''Run a command through subprocess ''' if type(command) is str: command_as_string = command command = shlex.split(command) else: command_as_string = ' ' .join(command) if len(command) == 0: raise click.ClickException('Can\'t run an empty command.') _kwargs = dict( stdin=subprocess.DEVNULL, # no interactions stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) _kwargs.update(kwargs) if stream: _kwargs['bufsize'] = 1 if log_command: log.debug('Running command', command=hide_secrets(command_as_string, secrets), kwargs=_kwargs) with subprocess.Popen(command, **_kwargs) as proc: if stream: output = [] for line in proc.stdout: line = line.decode('utf-8', 'ignore') line = line.rstrip('\n') if log_output: log.debug(hide_secrets(line, secrets)) output.append(line) if handle_stream_line: handle_stream_line(line) output = '\n'.join(output) # TODO: When needed we should also add possibility to stream stdout # and sterr separatly using asyncio.subprocess: # https://kevinmccarthy.org/2016/07/25/streaming-subprocess-stdin-and-stdout-with-asyncio-in-python/ # You can still pipe stderr into stdout which is enough for now. error = '' else: output, error = proc.communicate() return proc.returncode, output, error
def cmd(project, zsh, quiet, command, nix_shell, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): project_config = please_cli.config.PROJECTS_CONFIG.get(project, {}) run_type = project_config.get('run') run_options = project_config.get('run_options', {}) TMP_DIR = pathlib.Path(please_cli.config.TMP_DIR) ROOT_DIR = pathlib.Path(please_cli.config.ROOT_DIR) CERTS_DIR = TMP_DIR / 'certs' ROOT_NIX_FILE = ROOT_DIR / 'nix' / 'default.nix' CA_CERT_FILE = CERTS_DIR / 'ca.crt' SERVER_CERT_FILE = CERTS_DIR / 'server.crt' SERVER_KEY_FILE = CERTS_DIR / 'server.key' run = [] if zsh or command: run.append('--run') if command: run.append(command + '; exit') elif zsh: run.append('zsh') _command = [ nix_shell, f'{ROOT_NIX_FILE}', '-A', project, '-j', '1', ] + run envs = dict( SERVICES_ROOT=f'{ROOT_DIR}/', SSL_DEV_CA=f'{CERTS_DIR}', SSL_CACERT=f'{CA_CERT_FILE}', SSL_CERT=f'{SERVER_CERT_FILE}', SSL_KEY=f'{SERVER_KEY_FILE}', HOST=run_options.get('host', '127.0.0.1'), PORT=str(run_options.get('port', 8000)), RELEASE_VERSION=please_cli.config.VERSION, RELEASE_CHANNEL='development', PYTHONPATH='', TASKCLUSTER_SECRET=taskcluster_secret, ) if taskcluster_client_id: envs['TASKCLUSTER_CLIENT_ID'] = taskcluster_client_id if taskcluster_access_token: envs['TASKCLUSTER_ACCESS_TOKEN'] = taskcluster_access_token for require in project_config.get('requires', []): env_name = '{}_URL'.format(please_cli.utils.normalize_name(require).upper()) env_value = '{}://{}:{}'.format( please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'), please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', envs['HOST']), please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'], ) envs[env_name] = env_value for env_name, env_value in run_options.get('envs', {}).items(): env_name = please_cli.utils.normalize_name(env_name).upper() envs[env_name] = env_value click.echo(' => Setting environment variables:') for env_name, env_value in envs.items(): click.echo(f' - {env_name}="{env_value}"') os.environ[env_name] = env_value if command: handle_stream_line = None if quiet is False: handle_stream_line = click.echo return cli_common.command.run( _command, stream=True, handle_stream_line=handle_stream_line, stderr=subprocess.STDOUT, ) else: log.debug('Running command using os.system', command=_command) return os.system(' '.join(_command)) / 256, '', ''
def cmd(project, zsh, quiet, command, nix_shell, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): project_config = please_cli.config.PROJECTS_CONFIG.get(project, {}) # run_type = project_config.get('run') run_options = project_config.get('run_options', {}) TMP_DIR = pathlib.Path(please_cli.config.TMP_DIR) ROOT_DIR = pathlib.Path(please_cli.config.ROOT_DIR) CERTS_DIR = TMP_DIR / 'certs' ROOT_NIX_FILE = ROOT_DIR / 'nix' / 'default.nix' CA_CERT_FILE = CERTS_DIR / 'ca.crt' SERVER_CERT_FILE = CERTS_DIR / 'server.crt' SERVER_KEY_FILE = CERTS_DIR / 'server.key' run = [] if zsh or command: run.append('--run') if command: run.append(command + '; exit') elif zsh: run.append('zsh') _command = [ nix_shell, f'{ROOT_NIX_FILE}', '-A', project, '-j', '1', ] + run envs = dict( SERVICES_ROOT=f'{ROOT_DIR}/', SSL_DEV_CA=f'{CERTS_DIR}', SSL_CACERT=f'{CA_CERT_FILE}', SSL_CERT=f'{SERVER_CERT_FILE}', SSL_KEY=f'{SERVER_KEY_FILE}', HOST=run_options.get('host', os.environ.get('HOST', '127.0.0.1')), PORT=str(run_options.get('port', 8000)), RELEASE_VERSION=please_cli.config.VERSION, RELEASE_CHANNEL='development', PYTHONPATH='', TASKCLUSTER_SECRET=taskcluster_secret, ) if taskcluster_client_id: envs['TASKCLUSTER_CLIENT_ID'] = taskcluster_client_id if taskcluster_access_token: envs['TASKCLUSTER_ACCESS_TOKEN'] = taskcluster_access_token for require in project_config.get('requires', []): env_name = '{}_URL'.format(please_cli.utils.normalize_name(require).upper()) env_value = '{}://{}:{}'.format( please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'), please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', envs['HOST']), please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'], ) envs[env_name] = env_value for env_name, env_value in run_options.get('envs', {}).items(): env_name = please_cli.utils.normalize_name(env_name).upper() envs[env_name] = env_value click.echo(' => Setting environment variables:') for env_name, env_value in envs.items(): click.echo(f' - {env_name}="{env_value}"') os.environ[env_name] = env_value if command: handle_stream_line = None if quiet is False: handle_stream_line = click.echo return cli_common.command.run( _command, stream=True, handle_stream_line=handle_stream_line, stderr=subprocess.STDOUT, ) else: log.debug('Running command using os.system', command=_command) return os.system(' '.join(_command)) / 256, '', ''
def _notify_status_change(trees_changes, tags=[]): if flask.current_app.config.get('STATUSPAGE_ENABLE'): log.debug('Notify statuspage about trees changes.') components = flask.current_app.config.get('STATUSPAGE_COMPONENTS', {}) token = flask.current_app.config.get('STATUSPAGE_TOKEN') if not token: log.error('STATUSPAGE_PAGE_ID not defined in app config.') else: headers = {'Authorization': f'OAuth {token}'} for tree_change in trees_changes: tree, status_from, status_to = tree_change if tree.tree not in components.keys(): continue log.debug(f'Notify statuspage about: {tree.tree}') component_id = components[tree.tree] # create an accident if status_from in ['open', 'approval required' ] and status_to == 'closed': _statuspage_create_incident( headers, component_id, tree, status_from, status_to, ) # close an accident elif status_from == 'closed' and status_to in [ 'open', 'approval required' ]: _statuspage_resolve_incident( headers, component_id, tree, status_from, status_to, ) if flask.current_app.config.get('PULSE_TREESTATUS_ENABLE'): routing_key_pattern = 'tree/{0}/status_change' exchange = flask.current_app.config.get('PULSE_TREESTATUS_EXCHANGE') for tree_change in trees_changes: tree, status_from, status_to = tree_change payload = { 'status_from': status_from, 'status_to': status_to, 'tree': tree.to_dict(), 'tags': tags } routing_key = routing_key_pattern.format(tree.tree) log.info('Sending pulse to {} for tree: {}'.format( exchange, tree.tree, )) try: flask.current_app.pulse.publish(exchange, routing_key, payload) except Exception as e: import traceback msg = 'Can\'t send notification to pulse.' trace = traceback.format_exc() log.error('{0}\nException:{1}\nTraceback: {2}'.format( msg, e, trace)) # noqa