def gcp_set_account(account): """ Call gcloud to set current account :param account: gcloud account :return: True if successful otherwise False """ _logger.debug('setting authentication login to {0}'.format(account)) if not account: _logger.error('no GCP account given, aborting...') return False prog = which('gcloud') args = shlex.split('{0} auth login {1}'.format(prog, account)) # pylint: disable=W0612 pcode, so, se = run_external_program(args) if pcode != 0: _logger.error( 'failed to set gcp auth login account. ({0}: {1})'.format( pcode, so)) return False _logger.debug('successfully set gcp auth login account') return True
def gcp_activate_proxy(enable_sandbox=False, enable_test=False): """ Launch GCP sql proxy :param enable_sandbox: add proxy instance for sandbox :param enable_test: add proxy instance for test :return: Popen object """ prog = which('cloud_sql_proxy') # Set mysql proxy instances instances = '' instances += '{0}=tcp:9900,'.format(GCP_INSTANCES['all-of-us-rdr-prod']) instances += '{0}=tcp:9910,'.format(GCP_INSTANCES['all-of-us-rdr-stable']) instances += '{0}=tcp:9920,'.format(GCP_INSTANCES['all-of-us-rdr-staging']) if enable_sandbox is True: instances += '{0}=tcp:9930,'.format( GCP_INSTANCES['all-of-us-rdr-sandbox']) if enable_test is True: instances += '{0}=tcp:9940,'.format(GCP_INSTANCES['pmi-drc-api-test']) instances += '{0}=tcp:9945,'.format( GCP_INSTANCES['pmi-drc-api-test-repl']) # remove trailing comma instances = instances[:-1] p = subprocess.Popen( shlex.split('{0} -instances={1}'.format(prog, instances))) return p
def gcp_activate_sql_proxy(instances): """ Call cloud_sql_proxy to make a connection to the given instance. :param instances: full instance information, format "name:location:database=tcp:PORT, ...". :return: popen object """ prog = which('cloud_sql_proxy') p = subprocess.Popen( shlex.split('{0} -instances={1}'.format(prog, instances))) return p
def gcp_activate_sql_proxy(instances): """ Call cloud_sql_proxy to make a connection to the given instance. NOTE: If you are using a GCPProcessContext object, call self.gcp_env.activate_sql_proxy() instead of calling this function directly. :param instances: full instance information, format "name:location:database=tcp:PORT, ...". :return: popen object """ prog = which('cloud_sql_proxy') p = subprocess.Popen( shlex.split('{0} -instances={1} '.format(prog, instances))) if not p: raise IOError('failed to execute cloud_sql_proxy') return p
def gcp_test_environment(): """ Make sure the local environment is good :return: True if yes, False if not. """ progs = ['gcloud', 'gsutil', 'cloud_sql_proxy', 'grep'] for prog in progs: if not which(prog): _logger.error('[{0}] executable is not found.'.format(prog)) return False # TODO: Future: put additional checks here as needed, IE: required environment vars. _logger.info('local environment is good.') return True
def gcp_gsutil_command(cmd, args, flags=None): """ Run a gsutil command :param cmd: gsutil command name :param args: command arguments :param flags: additional flags to pass to gsutil executable :return: (exit code, stdout, stderr) """ if not cmd or not args or not isinstance(args, str): _logger.error('invalid parameters passed to gcp_gsutil_command.') return False prog = which('gsutil') args = shlex.split('{0} {1} {2} {3}'.format(prog, flags if flags else '', cmd, args)) return run_external_program(args)
def gcp_gcloud_command(group, args, flags=None): """ Run a gcloud command :param group: group name :param args: command arguments :param flags: additional flags to pass to gcloud executable :return: (exit code, stdout, stderr) """ if not group or not args or not isinstance(args, str): _logger.error('invalid parameters passed to gcp_gcloud_command.') return False prog = which('gcloud') args = shlex.split('{0} {1} {2} {3}'.format(prog, group, args, flags if flags else '')) return run_external_program(args)
def gcp_create_iam_service_creds(account, creds_account=None): """ # Note: Untested :param account: :param creds_account: :return: reference key """ # make sure key store directory exists if not os.path.exists(GCP_SERVICE_KEY_STORE): os.makedirs(GCP_SERVICE_KEY_STORE) # make sure we never duplicate an existing key while True: service_key = '{0}.json'.format(''.join( choice('0123456789ABCDEF') for _ in xrange(6))) service_key_file = os.path.join(GCP_SERVICE_KEY_STORE, service_key) if not os.path.exists( os.path.join(GCP_SERVICE_KEY_STORE, service_key_file)): break if creds_account is None: creds_account = account prog = which('gcloud') args = shlex.split( '{0} iam service-accounts keys create "{1}" --iam-account={2} --account={3}' .format(prog, service_key_file, account, creds_account)) # pylint: disable=W0612 pcode, so, se = run_external_program(args) if pcode != 0: _logger.error( 'failed to create iam service account key. ({0}: {1})'.format( pcode, so)) return False _logger.debug('successfully created iam service account key ({0})'.format( service_key)) return service_key
def gcp_delete_iam_service_creds(service_key, account, creds_account=None): """ # Note: Untested :param service_key: :param project: :param account: :param creds_account: :return: """ srv_key_file = os.path.join(GCP_SERVICE_KEY_STORE, service_key) if not os.path.exists(srv_key_file): _logger.error('specified iam service key does not exist ({0})'.format( service_key)) return False if creds_account is None: creds_account = account prog = which('gcloud') args = shlex.split( '{0} iam service-accounts keys delete "{1}" --iam-account={2} --account={3}' .format(prog, srv_key_file, account, creds_account)) # pylint: disable=W0612 pcode, so, se = run_external_program(args) if pcode != 0: _logger.error( 'failed to delete iam service account key. ({0}: {1})'.format( pcode, so)) return False _logger.debug('successfully deleted iam service account key ({0})'.format( service_key)) return service_key
def gcp_bq_command(cmd, args, global_flags=None, command_flags=None, headless=True): """ Run a bq command :param cmd: bq command name :param args: command arguments :param global_flags: global flags to pass to bq executable :param command_flags: command flags to pass to bq executable :param headless: run the 'bq' command in headless mode :return: (exit code, stdout, stderr) """ if not cmd or not args or not isinstance(args, str): _logger.error('invalid parameters passed to gcp_bq_command.') return False p_args = shlex.split('{0} {1} {2} {3} {4} {5}'.format( which('bq'), '--headless' if headless else '', global_flags if global_flags else '', cmd, command_flags if command_flags else '', args)) return run_external_program(p_args)
def run(): class TemplateDaemon(Daemon): stopProcessing = False def __init__(self, *args, **kwargs): self._args = kwargs.pop('args', None) super(TemplateDaemon, self).__init__(*args, **kwargs) def run(self): """ Main program process :return: Exit code value """ # Set SIGTERM signal Handler signal.signal(signal.SIGTERM, signal_term_handler) _logger.debug('signal handlers set.') result = gcp_set_account(args.account) if result is not True: _logger.error( 'failed to set authentication account, aborting.') return 1 po_proxy = gcp_activate_proxy(args.enable_sandbox, args.enable_test) if not po_proxy: _logger.error('cloud_sql_proxy process failed, aborting.') return 1 while self.stopProcessing is False: time.sleep(0.5) _logger.debug('stopping cloud_sql_proxy process...') po_proxy.kill() _logger.debug('stopped') return 0 def signal_term_handler(_sig, _frame): if not _daemon.stopProcessing: _logger.warning('received SIGTERM signal.') _daemon.stopProcessing = True # Set global debug value and setup application logging. setup_logging(_logger, progname, '--debug' in sys.argv) setup_unicode() # Setup program arguments. parser = argparse.ArgumentParser(prog=progname) # pylint: disable=E0602 parser.add_argument('--debug', help=_('Enable debug output'), default=False, action='store_true') # noqa # pylint: disable=E0602 parser.add_argument('--root-only', help=_('Must run as root user'), default=False, action='store_true') # noqa # pylint: disable=E0602 parser.add_argument('--nodaemon', help=_('Do not daemonize process'), default=False, action='store_true') # noqa # pylint: disable=E0602 parser.add_argument('--account', help=_('Security account')) # pylint: disable=E0602 parser.add_argument('--enable-sandbox', help=_('Add proxy to all-of-us-rdr-sandbox'), default=False, action='store_true') # noqa # pylint: disable=E0602 parser.add_argument('--enable-test', help=_('Add proxy to pmi-drc-api-test'), default=False, action='store_true') # noqa parser.add_argument('action', choices=('start', 'stop', 'restart'), default='') # noqa args = parser.parse_args() if args.root_only is True and os.getuid() != 0: _logger.warning('daemon must be run as root') sys.exit(4) if is_valid_email(args.account) is False: if 'RDR_ACCOUNT' not in os.environ: _logger.error( 'account parameter is invalid and RDR_ACCOUNT shell var is not set.' ) return 1 else: args.account = os.environ['RDR_ACCOUNT'] if which('cloud_sql_proxy') is None: _logger.error( 'cloud_sql_proxy executable not found, ' + 'create symlink to cloud_sql_proxy in /usr/local/bin/ directory') # --nodaemon only valid with start action if args.nodaemon and args.action != 'start': print( '{0}: error: --nodaemon option not valid with stop or restart action' .format(progname)) sys.exit(1) _logger.info(' account: {0}'.format(args.account)) if args.action == 'start': _logger.info(' tcp: 127.0.0.1:9900 -> all-of-us-rdr-prod') _logger.info(' tcp: 127.0.0.1:9910 -> all-of-us-rdr-stable') _logger.info(' tcp: 127.0.0.1:9920 -> all-of-us-rdr-staging') if args.enable_sandbox is True: _logger.info( ' tcp: 127.0.0.1:9930 -> all-of-us-rdr-sandbox') if args.enable_test: _logger.info(' tcp: 127.0.0.1:9940 -> all-of-us-rdr-test') # Do not fork the daemon process for systemd service or debugging, run in foreground. if args.nodaemon is True: if args.action == 'start': _logger.info('running daemon in foreground.') _daemon = TemplateDaemon(args=args) _daemon.run() else: pidpath = os.path.expanduser('~/.local/run') pidfile = os.path.join(pidpath, '{0}.pid'.format(progname)) logpath = os.path.expanduser('~/.local/log') logfile = os.path.join(logpath, '{0}.log'.format(progname)) if args.action == 'start': _logger.info('running daemon in background.') # Setup daemon object # make sure PID and Logging path exist if not os.path.exists(pidpath): os.makedirs(pidpath) if not os.path.exists(logpath): os.makedirs(logpath) _daemon = TemplateDaemon(procbase='', dirmask='0o700', pidfile=pidfile, uid='root', gid='root', stdin='/dev/null', stdout=logfile, stderr=logfile, args=args) if args.action == 'start': _logger.debug('Starting daemon.') _daemon.start() elif args.action == 'stop': _logger.debug('Stopping daemon.') _daemon.stop() elif args.action == 'restart': _logger.debug('Restarting daemon.') _daemon.restart() return 0