def gcp_set_account(account):
"""
Call gcloud to set current account
:param account: gcloud account
:return: True if successful otherwise False
"""
_logger.debug('setting authentication login to {0}'.format(account))
if not account:
_logger.error('no GCP account given, aborting...')
return False
prog = which('gcloud')
args = shlex.split('{0} auth login {1}'.format(prog, account))
# pylint: disable=W0612
pcode, so, se = run_external_program(args)
if pcode != 0:
_logger.error(
'failed to set gcp auth login account. ({0}: {1})'.format(
pcode, so))
return False
_logger.debug('successfully set gcp auth login account')
return True
def gcp_activate_proxy(enable_sandbox=False, enable_test=False):
"""
Launch GCP sql proxy
:param enable_sandbox: add proxy instance for sandbox
:param enable_test: add proxy instance for test
:return: Popen object
"""
prog = which('cloud_sql_proxy')
# Set mysql proxy instances
instances = ''
instances += '{0}=tcp:9900,'.format(GCP_INSTANCES['all-of-us-rdr-prod'])
instances += '{0}=tcp:9910,'.format(GCP_INSTANCES['all-of-us-rdr-stable'])
instances += '{0}=tcp:9920,'.format(GCP_INSTANCES['all-of-us-rdr-staging'])
if enable_sandbox is True:
instances += '{0}=tcp:9930,'.format(
GCP_INSTANCES['all-of-us-rdr-sandbox'])
if enable_test is True:
instances += '{0}=tcp:9940,'.format(GCP_INSTANCES['pmi-drc-api-test'])
instances += '{0}=tcp:9945,'.format(
GCP_INSTANCES['pmi-drc-api-test-repl'])
# remove trailing comma
instances = instances[:-1]
p = subprocess.Popen(
shlex.split('{0} -instances={1}'.format(prog, instances)))
return p
def gcp_activate_sql_proxy(instances):
"""
Call cloud_sql_proxy to make a connection to the given instance.
:param instances: full instance information, format "name:location:database=tcp:PORT, ...".
:return: popen object
"""
prog = which('cloud_sql_proxy')
p = subprocess.Popen(
shlex.split('{0} -instances={1}'.format(prog, instances)))
return p
def gcp_activate_sql_proxy(instances):
"""
Call cloud_sql_proxy to make a connection to the given instance.
NOTE: If you are using a GCPProcessContext object, call self.gcp_env.activate_sql_proxy() instead
of calling this function directly.
:param instances: full instance information, format "name:location:database=tcp:PORT, ...".
:return: popen object
"""
prog = which('cloud_sql_proxy')
p = subprocess.Popen(
shlex.split('{0} -instances={1} '.format(prog, instances)))
if not p:
raise IOError('failed to execute cloud_sql_proxy')
return p
def gcp_test_environment():
"""
Make sure the local environment is good
:return: True if yes, False if not.
"""
progs = ['gcloud', 'gsutil', 'cloud_sql_proxy', 'grep']
for prog in progs:
if not which(prog):
_logger.error('[{0}] executable is not found.'.format(prog))
return False
# TODO: Future: put additional checks here as needed, IE: required environment vars.
_logger.info('local environment is good.')
return True
def gcp_gsutil_command(cmd, args, flags=None):
"""
Run a gsutil command
:param cmd: gsutil command name
:param args: command arguments
:param flags: additional flags to pass to gsutil executable
:return: (exit code, stdout, stderr)
"""
if not cmd or not args or not isinstance(args, str):
_logger.error('invalid parameters passed to gcp_gsutil_command.')
return False
prog = which('gsutil')
args = shlex.split('{0} {1} {2} {3}'.format(prog, flags if flags else '',
cmd, args))
return run_external_program(args)
def gcp_gcloud_command(group, args, flags=None):
"""
Run a gcloud command
:param group: group name
:param args: command arguments
:param flags: additional flags to pass to gcloud executable
:return: (exit code, stdout, stderr)
"""
if not group or not args or not isinstance(args, str):
_logger.error('invalid parameters passed to gcp_gcloud_command.')
return False
prog = which('gcloud')
args = shlex.split('{0} {1} {2} {3}'.format(prog, group, args,
flags if flags else ''))
return run_external_program(args)
def gcp_create_iam_service_creds(account, creds_account=None):
"""
# Note: Untested
:param account:
:param creds_account:
:return: reference key
"""
# make sure key store directory exists
if not os.path.exists(GCP_SERVICE_KEY_STORE):
os.makedirs(GCP_SERVICE_KEY_STORE)
# make sure we never duplicate an existing key
while True:
service_key = '{0}.json'.format(''.join(
choice('0123456789ABCDEF') for _ in xrange(6)))
service_key_file = os.path.join(GCP_SERVICE_KEY_STORE, service_key)
if not os.path.exists(
os.path.join(GCP_SERVICE_KEY_STORE, service_key_file)):
break
if creds_account is None:
creds_account = account
prog = which('gcloud')
args = shlex.split(
'{0} iam service-accounts keys create "{1}" --iam-account={2} --account={3}'
.format(prog, service_key_file, account, creds_account))
# pylint: disable=W0612
pcode, so, se = run_external_program(args)
if pcode != 0:
_logger.error(
'failed to create iam service account key. ({0}: {1})'.format(
pcode, so))
return False
_logger.debug('successfully created iam service account key ({0})'.format(
service_key))
return service_key
def gcp_delete_iam_service_creds(service_key, account, creds_account=None):
"""
# Note: Untested
:param service_key:
:param project:
:param account:
:param creds_account:
:return:
"""
srv_key_file = os.path.join(GCP_SERVICE_KEY_STORE, service_key)
if not os.path.exists(srv_key_file):
_logger.error('specified iam service key does not exist ({0})'.format(
service_key))
return False
if creds_account is None:
creds_account = account
prog = which('gcloud')
args = shlex.split(
'{0} iam service-accounts keys delete "{1}" --iam-account={2} --account={3}'
.format(prog, srv_key_file, account, creds_account))
# pylint: disable=W0612
pcode, so, se = run_external_program(args)
if pcode != 0:
_logger.error(
'failed to delete iam service account key. ({0}: {1})'.format(
pcode, so))
return False
_logger.debug('successfully deleted iam service account key ({0})'.format(
service_key))
return service_key
def gcp_bq_command(cmd,
args,
global_flags=None,
command_flags=None,
headless=True):
"""
Run a bq command
:param cmd: bq command name
:param args: command arguments
:param global_flags: global flags to pass to bq executable
:param command_flags: command flags to pass to bq executable
:param headless: run the 'bq' command in headless mode
:return: (exit code, stdout, stderr)
"""
if not cmd or not args or not isinstance(args, str):
_logger.error('invalid parameters passed to gcp_bq_command.')
return False
p_args = shlex.split('{0} {1} {2} {3} {4} {5}'.format(
which('bq'), '--headless' if headless else '',
global_flags if global_flags else '', cmd,
command_flags if command_flags else '', args))
return run_external_program(p_args)
def run():
class TemplateDaemon(Daemon):
stopProcessing = False
def __init__(self, *args, **kwargs):
self._args = kwargs.pop('args', None)
super(TemplateDaemon, self).__init__(*args, **kwargs)
def run(self):
"""
Main program process
:return: Exit code value
"""
# Set SIGTERM signal Handler
signal.signal(signal.SIGTERM, signal_term_handler)
_logger.debug('signal handlers set.')
result = gcp_set_account(args.account)
if result is not True:
_logger.error(
'failed to set authentication account, aborting.')
return 1
po_proxy = gcp_activate_proxy(args.enable_sandbox,
args.enable_test)
if not po_proxy:
_logger.error('cloud_sql_proxy process failed, aborting.')
return 1
while self.stopProcessing is False:
time.sleep(0.5)
_logger.debug('stopping cloud_sql_proxy process...')
po_proxy.kill()
_logger.debug('stopped')
return 0
def signal_term_handler(_sig, _frame):
if not _daemon.stopProcessing:
_logger.warning('received SIGTERM signal.')
_daemon.stopProcessing = True
# Set global debug value and setup application logging.
setup_logging(_logger, progname, '--debug' in sys.argv)
setup_unicode()
# Setup program arguments.
parser = argparse.ArgumentParser(prog=progname)
# pylint: disable=E0602
parser.add_argument('--debug',
help=_('Enable debug output'),
default=False,
action='store_true') # noqa
# pylint: disable=E0602
parser.add_argument('--root-only',
help=_('Must run as root user'),
default=False,
action='store_true') # noqa
# pylint: disable=E0602
parser.add_argument('--nodaemon',
help=_('Do not daemonize process'),
default=False,
action='store_true') # noqa
# pylint: disable=E0602
parser.add_argument('--account', help=_('Security account'))
# pylint: disable=E0602
parser.add_argument('--enable-sandbox',
help=_('Add proxy to all-of-us-rdr-sandbox'),
default=False,
action='store_true') # noqa
# pylint: disable=E0602
parser.add_argument('--enable-test',
help=_('Add proxy to pmi-drc-api-test'),
default=False,
action='store_true') # noqa
parser.add_argument('action',
choices=('start', 'stop', 'restart'),
default='') # noqa
args = parser.parse_args()
if args.root_only is True and os.getuid() != 0:
_logger.warning('daemon must be run as root')
sys.exit(4)
if is_valid_email(args.account) is False:
if 'RDR_ACCOUNT' not in os.environ:
_logger.error(
'account parameter is invalid and RDR_ACCOUNT shell var is not set.'
)
return 1
else:
args.account = os.environ['RDR_ACCOUNT']
if which('cloud_sql_proxy') is None:
_logger.error(
'cloud_sql_proxy executable not found, ' +
'create symlink to cloud_sql_proxy in /usr/local/bin/ directory')
# --nodaemon only valid with start action
if args.nodaemon and args.action != 'start':
print(
'{0}: error: --nodaemon option not valid with stop or restart action'
.format(progname))
sys.exit(1)
_logger.info(' account: {0}'.format(args.account))
if args.action == 'start':
_logger.info(' tcp: 127.0.0.1:9900 -> all-of-us-rdr-prod')
_logger.info(' tcp: 127.0.0.1:9910 -> all-of-us-rdr-stable')
_logger.info(' tcp: 127.0.0.1:9920 -> all-of-us-rdr-staging')
if args.enable_sandbox is True:
_logger.info(
' tcp: 127.0.0.1:9930 -> all-of-us-rdr-sandbox')
if args.enable_test:
_logger.info(' tcp: 127.0.0.1:9940 -> all-of-us-rdr-test')
# Do not fork the daemon process for systemd service or debugging, run in foreground.
if args.nodaemon is True:
if args.action == 'start':
_logger.info('running daemon in foreground.')
_daemon = TemplateDaemon(args=args)
_daemon.run()
else:
pidpath = os.path.expanduser('~/.local/run')
pidfile = os.path.join(pidpath, '{0}.pid'.format(progname))
logpath = os.path.expanduser('~/.local/log')
logfile = os.path.join(logpath, '{0}.log'.format(progname))
if args.action == 'start':
_logger.info('running daemon in background.')
# Setup daemon object
# make sure PID and Logging path exist
if not os.path.exists(pidpath):
os.makedirs(pidpath)
if not os.path.exists(logpath):
os.makedirs(logpath)
_daemon = TemplateDaemon(procbase='',
dirmask='0o700',
pidfile=pidfile,
uid='root',
gid='root',
stdin='/dev/null',
stdout=logfile,
stderr=logfile,
args=args)
if args.action == 'start':
_logger.debug('Starting daemon.')
_daemon.start()
elif args.action == 'stop':
_logger.debug('Stopping daemon.')
_daemon.stop()
elif args.action == 'restart':
_logger.debug('Restarting daemon.')
_daemon.restart()
return 0
