def init():
"""Initialize the database for local development."""
click.echo(click.style(">>>> Initialization", fg='magenta', bold=True))
params = dict(db_name=database.DATABASE_NAME,
db_user=database.DATABASE_USER,
db_pass=database.DATABASE_PASSWORD)
components = [
(
"Create database",
"echo \"CREATE DATABASE IF NOT EXISTS {db_name} CHARACTER SET utf8;\" | mysql -u root"
.format(**params),
),
(
"Grant privileges to {db_user}".format(**params),
"echo \"GRANT ALL PRIVILEGES ON {db_user}.* TO '{db_user}'@'localhost' IDENTIFIED BY '{db_pass}';\" | mysql -u root"
.format(**params),
),
(
"Cleaning",
"echo \"FLUSH PRIVILEGES;\" | mysql -u root".format(**params),
),
(
"Create local service account file from a template",
"cp backends/data/service-account.json.example backends/data/service-account.json",
)
]
for component in components:
name, cmd = component
shared.execute_command(name, cmd, cwd=constants.PROJECT_DIR)
click.echo(click.style("Done.", fg='magenta', bold=True))
def deploy_controller(stage, debug=False):
project_id = stage.project_id_gae
cloud_db_uri = stage.cloud_db_uri
pubsub_verification_token = stage.pubsub_verification_token
cmds = [
'cp .gcloudignore-controller .gcloudignore',
'cp requirements-controller.txt requirements.txt',
'cp controller_app.yaml controller_app_with_env_vars.yaml',
'\n'.join([
'cat >> controller_app_with_env_vars.yaml <<EOL',
'env_variables:',
f' PUBSUB_VERIFICATION_TOKEN: {pubsub_verification_token}',
f' DATABASE_URI: {cloud_db_uri}',
'EOL',
]),
(f' {GCLOUD} app deploy controller_app_with_env_vars.yaml'
f' --version=v1 --project={project_id}'),
]
cmd_workdir = os.path.join(stage.workdir, 'backend')
total = len(cmds)
for i, cmd in enumerate(cmds):
shared.execute_command(f'Deploy controller service ({i + 1}/{total})',
cmd,
cwd=cmd_workdir,
debug=debug)
def do_requirements(debug):
"""Install required Python packages."""
click.echo(
click.style(">>>> Install requirements", fg='magenta', bold=True))
components = [
(
"Install interface backend requirements",
"pip install -r ibackend/requirements.txt -t lib",
),
(
"Install jobs backend requirements",
"pip install -r jbackend/requirements.txt -t lib",
),
(
"Install documentation requirements",
"pip install \"sphinx==1.7.2\" \"sphinx-autobuild==0.7.1\"",
)
]
for component in components:
name, cmd = component
shared.execute_command(name,
cmd,
cwd=constants.BACKENDS_DIR,
debug=debug)
click.echo(click.style("Done.", fg='magenta', bold=True))
def start_cloud_sql_proxy(stage, debug=False):
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
commands = [
(
"mkdir -p {cloudsql_dir}".format(cloudsql_dir=stage.cloudsql_dir),
False,
),
(
"echo \"CLOUD_SQL_PROXY=$CLOUD_SQL_PROXY\"",
False,
),
(
"$CLOUD_SQL_PROXY -projects={project_id} -instances={db_instance_conn_name} -dir={cloudsql_dir} 2>/dev/null &".format(
project_id=stage.project_id_gae,
cloudsql_dir=stage.cloudsql_dir,
db_instance_conn_name=stage.db_instance_conn_name),
True,
),
(
"sleep 5", # Wait for cloud_sql_proxy to start.
False
),
]
total = len(commands)
idx = 1
for comp in commands:
cmd, force_std_out = comp
shared.execute_command("Start CloudSQL proxy (%d/%d)" % (idx, total),
cmd,
cwd='.',
force_std_out=force_std_out,
debug=debug)
idx += 1
def setup():
"""Prepare the environment before deployment."""
click.echo(click.style(">>>> Setup local env", fg='magenta', bold=True))
components = [
("Homebrew", "command -v brew",
"echo Please execute the following command first:\n\'/usr/bin/ruby " +
"-e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\"\'",
is_executable_file),
("Node.js", "command -v node", "brew install node",
is_executable_file),
("Angular", "command -v ng", "npm install -g @angular/cli",
is_executable_file),
("MySQL", "command -v mysql", "brew install mysql",
is_executable_file),
("Google Cloud SDK", "command -v gcloud",
"export CLOUDSDK_CORE_DISABLE_PROMPTS=1 && \
curl https://sdk.cloud.google.com | bash", is_executable_file),
("App Engine Python", "gcloud --version | grep \"app-engine-python\"",
"gcloud components install app-engine-python", is_not_empty),
]
for component in components:
step_name, check_cmd, install_cmd, check_cmd_res_func = component
status, out, err = shared.execute_command("Check %s" % step_name,
check_cmd)
if status == 0 and check_cmd_res_func(out.strip()):
click.echo(" Already installed.")
else:
shared.execute_command("Install %s" % step_name, install_cmd)
click.echo(click.style("Done.", fg='magenta', bold=True))
def _get_regions(project_id):
gcloud = '$GOOGLE_CLOUD_SDK/bin/gcloud --quiet'
cmd = f'{gcloud} app describe --verbosity critical --project={project_id}'
cmd += '| grep locationId'
status, out, err = shared.execute_command('Get App Engine region',
cmd,
stream_output_in_debug=False)
if status == 0: # App Engine app had already been deployed in some region.
region = out.strip().split()[1]
else: # Get the list of available App Engine regions and prompt user.
click.echo(' No App Engine app has been deployed yet.')
cmd = f"{gcloud} app regions list --format='value(region)'"
status, out, err = shared.execute_command(
'Get available App Engine regions',
cmd,
stream_output_in_debug=False)
regions = out.strip().split('\n')
for i, region in enumerate(regions):
click.echo(f'{i + 1}) {region}')
i = -1
while i < 0 or i >= len(regions):
i = click.prompt(
'Enter an index of the region to deploy CRMint in',
type=int) - 1
region = regions[i]
sql_region = region if region[-1].isdigit() else f'{region}1'
return region, sql_region
def copy_src_to_workdir(stage, debug=False):
copy_src_cmd = "rsync -r --delete \
--exclude=.git \
--exclude=.idea \
--exclude='*.pyc' \
--exclude=frontend/node_modules \
--exclude=backends/data/*.json . {workdir}".format(
workdir=stage.workdir)
copy_insight_config_cmd = "cp backends/data/insight.json {workdir}/backends/data/insight.json".format(
workdir=stage.workdir)
copy_service_account_cmd = "cp backends/data/{service_account_filename} {workdir}/backends/data/service-account.json".format(
workdir=stage.workdir,
service_account_filename=stage.service_account_file)
copy_db_conf = "echo \'SQLALCHEMY_DATABASE_URI=\"{cloud_db_uri}\"\' > {workdir}/backends/instance/config.py".format(
workdir=stage.workdir,
cloud_db_uri=stage.cloud_db_uri)
copy_app_data = """
cat > %(workdir)s/backends/data/app.json <<EOL
{
"notification_sender_email": "%(notification_sender_email)s",
"app_title": "%(app_title)s"
}
EOL""".strip() % dict(
workdir=stage.workdir,
app_title=stage.app_title,
notification_sender_email=stage.notification_sender_email)
# We dont't use prod environment for the frontend to speed up deploy.
copy_prod_env = """
cat > %(workdir)s/frontend/src/environments/environment.ts <<EOL
export const environment = {
production: true,
app_title: "%(app_title)s",
enabled_stages: %(enabled_stages)s
}
EOL""".strip() % dict(
workdir=stage.workdir,
app_title=stage.app_title,
enabled_stages="true" if stage.enabled_stages else "false")
commands = [
copy_src_cmd,
copy_insight_config_cmd,
copy_service_account_cmd,
copy_db_conf,
copy_app_data,
copy_prod_env,
]
total = len(commands)
idx = 1
for cmd in commands:
shared.execute_command("Copy source code to working directory (%d/%d)" % (idx, total),
cmd,
cwd=constants.PROJECT_DIR,
debug=debug)
idx += 1
def create_pubsub_subscriptions(stage, debug=False):
existing_subscriptions = _get_existing_pubsub_entities(
stage, 'subscriptions', debug)
project_id = stage.project_id_gae
service_account = f'{project_id}@appspot.gserviceaccount.com'
for topic_id in SUBSCRIPTIONS:
subscription_id = f'{topic_id}-subscription'
if subscription_id in existing_subscriptions:
click.echo(
f' PubSub subscription {subscription_id} already exists')
continue
subscription = SUBSCRIPTIONS[topic_id]
if subscription is None:
continue
path = subscription['path']
token = stage.pubsub_verification_token
push_endpoint = f'https://{project_id}.appspot.com/{path}?token={token}'
ack_deadline = subscription['ack_deadline_seconds']
minimum_backoff = subscription['minimum_backoff']
min_retry_delay = f'{minimum_backoff}s'
cmd = (
f' {GCLOUD} --project={project_id} pubsub subscriptions create'
f' {subscription_id} --topic={topic_id} --topic-project={project_id}'
f' --ack-deadline={ack_deadline} --min-retry-delay={min_retry_delay}'
f' --expiration-period=never --push-endpoint={push_endpoint}'
f' --push-auth-service-account={service_account}')
shared.execute_command(
f'Creating PubSub subscription {subscription_id}',
cmd,
debug=debug)
def deploy_backends(stage, debug=False):
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
commands = [
". env/bin/activate && {gcloud_bin} --project={project_id} app deploy gae_ibackend.yaml --version=v1".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae),
". env/bin/activate && {gcloud_bin} --project={project_id} app deploy gae_jbackend.yaml --version=v1".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae),
". env/bin/activate && {gcloud_bin} --project={project_id} app deploy cron.yaml".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae),
". env/bin/activate && {gcloud_bin} --project={project_id} app deploy \"{workdir}/frontend/dispatch.yaml\"".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae,
workdir=stage.workdir),
]
cmd_workdir = os.path.join(stage.workdir, 'backends')
total = len(commands)
idx = 1
for cmd in commands:
shared.execute_command("Deploy backend services (%d/%d)" % (idx, total),
cmd,
cwd=cmd_workdir,
debug=debug)
idx += 1
def _run_flask_command(stage, step_name, flask_command_name="--help", debug=False):
cmd_workdir = os.path.join(stage.workdir, 'backends')
command = ". env/bin/activate && python -m flask {command_name}".format(
command_name=flask_command_name)
shared.execute_command(step_name,
command,
cwd=cmd_workdir,
debug=debug)
def deploy_dispatch_rules(stage, debug=False):
project_id = stage.project_id_gae
cmd = f' {GCLOUD} --project={project_id} app deploy dispatch.yaml'
cmd_workdir = os.path.join(stage.workdir, 'frontend')
shared.execute_command('Deploy dispatch rules',
cmd,
cwd=cmd_workdir,
debug=debug)
def create_appengine(stage, debug=False):
if _check_if_appengine_instance_exists(stage, debug=debug):
click.echo(' App Engine app already exists.')
return
project_id = stage.project_id_gae
region = stage.project_region
cmd = f'{GCLOUD} app create --project={project_id} --region={region}'
shared.execute_command('Create App Engine instance', cmd, debug=debug)
def now(stage_name, debug):
"""Generate Vertex AI pipelines."""
msg = click.style(
" _ _ ___________ _____ _______ __ ___ _____ _ _ _____ _ _\n",
fg='bright_blue')
msg += click.style(
"| | | | ___| ___ \_ _| ___\ \ / / / _ \|_ _| | \ | | _ || | | |\n",
fg='bright_blue')
msg += click.style(
"| | | | |__ | |_/ / | | | |__ \ V / / /_\ \ | | | \| | | | || | | |\n",
fg='bright_red')
msg += click.style(
"| | | | __|| / | | | __| / \ | _ | | | | . ` | | | || |/\| |\n",
fg='bright_red')
msg += click.style(
"\ \_/ / |___| |\ \ | | | |___/ /^\ \ | | | |_| |_ | |\ \ \_/ /\ /\ /\n",
fg='bright_yellow')
msg += click.style(
" \___/\____/\_| \_| \_/ \____/\/ \/ \_| |_/\___/ \_| \_/\___/ \/ \/\n",
fg='bright_green')
click.echo(msg)
stage_name, stage = cloud.fetch_stage_or_default(stage_name,
debug=debug,
silent_step_name=True)
stage = shared.before_hook(stage, stage_name)
platforms = ['GA4', 'Universal Analytics']
click.echo(
'Vertex AI Now is available for both GA4 & Universal Analytics\n'
'Google Analytics property types.\n'
'--------------------------------------------')
for i, p in enumerate(platforms):
click.echo(f'{i + 1}) {p}')
ind = click.prompt(
'Enter the index for the Google Analytics property type', type=int) - 1
platform = platforms[ind]
if platform == 'GA4':
training_file, prediction_file = pipelines._get_ga4_config(stage,
ml='vertex')
if platform == 'Universal Analytics':
training_file, prediction_file = pipelines._get_ua_config(stage,
ml='vertex')
local_db_uri = stage.local_db_uri
env_vars = f'DATABASE_URI="{local_db_uri}" FLASK_APP=controller_app.py'
cloud.install_required_packages(stage)
cloud.display_workdir(stage)
cloud.copy_src_to_workdir(stage)
cloud.download_cloud_sql_proxy(stage)
cloud.start_cloud_sql_proxy(stage)
cloud.install_python_packages(stage)
cmd_workdir = os.path.join(stage.workdir, 'backend')
cmd = (' . .venv_controller/bin/activate &&'
f' {env_vars} python -m flask import-pipelines {training_file} &&'
f' {env_vars} python -m flask import-pipelines {prediction_file}')
shared.execute_command('Importing training & prediction pipelines',
cmd,
cwd=cmd_workdir,
debug=debug)
cloud.stop_cloud_sql_proxy(stage)
def download_config_files(stage, debug=False):
stage_file_path = shared.get_stage_file(stage.stage_name)
service_account_file_path = shared.get_service_account_file(stage)
command = "cloudshell download-files \
\"{stage_file}\" \
\"{service_account_file}\"".format(
stage_file=stage_file_path,
service_account_file=service_account_file_path)
shared.execute_command("Download configuration files", command, debug=debug)
def grant_app_engine_default_service_account_permissions(stage, debug=False):
project_id = stage.project_id_gae
cmd = (
f'{GCLOUD} projects add-iam-policy-binding {project_id}'
f' --member="serviceAccount:{project_id}@appspot.gserviceaccount.com"'
f' --role="roles/editor"')
shared.execute_command(
"Grant App Engine default service account permissions",
cmd,
debug=debug)
def create_cloudsql_database_if_needed(stage, debug=False):
if _check_if_cloudsql_database_exists(stage, debug=debug):
click.echo(' CloudSQL database already exists.')
return
project_id = stage.project_id_gae
db_instance_name = stage.db_instance_name
db_name = stage.db_name
cmd = (f' {GCLOUD} sql databases create {db_name}'
f' --instance={db_instance_name} --project={project_id}')
shared.execute_command('Creating CloudSQL database', cmd, debug=debug)
def run_reset_pipelines(stage, debug=False):
local_db_uri = stage.local_db_uri
env_vars = f'DATABASE_URI="{local_db_uri}" FLASK_APP=controller_app.py'
cmd = (' . .venv_controller/bin/activate &&'
f' {env_vars} python -m flask reset-pipelines')
cmd_workdir = os.path.join(stage.workdir, 'backend')
shared.execute_command('Reset statuses of jobs and pipelines',
cmd,
cwd=cmd_workdir,
debug=debug)
def create_appengine(stage, debug=False):
if _check_if_appengine_instance_exists(stage, debug=debug):
click.echo(" App Engine already exists.")
return
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
command = "{gcloud_bin} app create --project={project_id} --region={region}".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae,
region=stage.project_region)
shared.execute_command("Create the App Engine instance", command, debug=debug)
def run_db_migrations(stage, debug=False):
local_db_uri = stage.local_db_uri
env_vars = f'DATABASE_URI="{local_db_uri}" FLASK_APP=controller_app.py'
cmd = (' . .venv_controller/bin/activate &&'
f' {env_vars} python -m flask db upgrade &&'
f' {env_vars} python -m flask db-seeds')
cmd_workdir = os.path.join(stage.workdir, 'backend')
shared.execute_command('Applying database migrations',
cmd,
cwd=cmd_workdir,
debug=debug)
def deploy_dispatch_rules(stage, debug=False):
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
# NB: Limit the node process memory usage to avoid overloading
# the Cloud Shell VM memory which makes it unresponsive.
command = "{gcloud_bin} --project={project_id} app deploy dispatch.yaml".format(
gcloud_bin=gcloud_command, project_id=stage.project_id_gae)
cmd_workdir = os.path.join(stage.workdir, 'frontend')
shared.execute_command("Deploy the dispatch.yaml rules",
command,
cwd=cmd_workdir,
debug=debug)
def install_required_packages(_, debug=False):
cmds = [
'mkdir -p ~/.cloudshell',
'> ~/.cloudshell/no-apt-get-warning',
'sudo apt-get install -y rsync libmysqlclient-dev python3-venv',
]
total = len(cmds)
for i, cmd in enumerate(cmds):
shared.execute_command(f'Install required packages ({i + 1}/{total})',
cmd,
debug=debug)
def create_scheduler_job(stage, debug=False):
if _check_if_scheduler_job_exists(stage, debug=debug):
click.echo(' Cloud Scheduler job already exists.')
return
project_id = stage.project_id_gae
cmd = (f' {GCLOUD} scheduler jobs create pubsub crmint-cron'
f" --project={project_id} --schedule='* * * * *'"
f' --topic=crmint-start-pipeline'
f' --message-body=\'{{"pipeline_ids": "scheduled"}}\''
f' --attributes="start_time=0" --description="CRMint\'s cron job"')
shared.execute_command('Create Cloud Scheduler job', cmd, debug=debug)
def grant_pubsub_permissions(stage, debug=False):
project_id = stage.project_id_gae
project_number = _get_project_number(stage, debug)
pubsub_sa = f'service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com'
cmd = (f' {GCLOUD} projects add-iam-policy-binding {project_id}'
f' --member="serviceAccount:{pubsub_sa}"'
f' --role="roles/iam.serviceAccountTokenCreator"')
shared.execute_command(
"Granting Cloud Pub/Sub the permission to create tokens",
cmd,
debug=debug)
def create_pubsub_topics(stage, debug=False):
existing_topics = _get_existing_pubsub_entities(stage, 'topics', debug)
crmint_topics = SUBSCRIPTIONS.keys()
topics_to_create = [t for t in crmint_topics if t not in existing_topics]
if not topics_to_create:
click.echo(" CRMint's PubSub topics already exist")
return
project_id = stage.project_id_gae
topics = ' '.join(topics_to_create)
cmd = f'{GCLOUD} --project={project_id} pubsub topics create {topics}'
shared.execute_command("Creating CRMint's PubSub topics", cmd, debug=debug)
def install_required_packages(stage, debug=False):
commands = [
"mkdir -p ~/.cloudshell",
"> ~/.cloudshell/no-apt-get-warning",
"sudo apt-get install -y rsync libmysqlclient-dev",
]
total = len(commands)
idx = 1
for cmd in commands:
shared.execute_command("Install required packages (%d/%d)" % (idx, total),
cmd,
debug=debug)
idx += 1
def create_cloudsql_instance_if_needed(stage, debug=False):
if _check_if_cloudsql_instance_exists(stage, debug=debug):
click.echo(' CloudSQL instance already exists.')
return
db_instance_name = stage.db_instance_name
project_id = stage.project_id_gae
project_sql_region = stage.project_sql_region
project_sql_tier = stage.project_sql_tier
cmd = (f' {GCLOUD} sql instances create {db_instance_name}'
f' --tier={project_sql_tier} --region={project_sql_region}'
f' --project={project_id} --database-version MYSQL_5_7'
f' --storage-auto-increase')
shared.execute_command("Creating a CloudSQL instance", cmd, debug=debug)
def create_mysql_database_if_needed(stage, debug=False):
if _check_if_mysql_database_exists(stage, debug=debug):
click.echo(" MySQL database already exists.")
return
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
command = "{gcloud_bin} sql databases create {db_name} \
--instance={db_instance_name} \
--project={project_id}".format(gcloud_bin=gcloud_command,
project_id=stage.project_id_gae,
db_instance_name=stage.db_instance_name,
db_name=stage.db_name)
shared.execute_command("Creating MySQL database", command, debug=debug)
def grant_cloud_build_permissions(stage, debug=False):
project_id = stage.project_id_gae
cmd = (f'{GCLOUD} projects list '
f' --filter="{project_id}" '
f' --format="value(PROJECT_NUMBER)"')
_, out, _ = shared.execute_command("Getting the project number",
cmd,
debug=debug)
project_number = out.strip()
cmd = (
f'{GCLOUD} projects add-iam-policy-binding {project_id}'
f' --member="serviceAccount:{project_number}@cloudbuild.gserviceaccount.com"'
f' --role="roles/storage.objectViewer"')
shared.execute_command("Grant Cloud Build permissions", cmd, debug=debug)
def copy_src_to_workdir(stage, debug=False):
workdir = stage.workdir
app_title = stage.app_title
notification_sender_email = stage.notification_sender_email
enabled_stages = 'true' if stage.enabled_stages else 'false'
copy_src_cmd = (f' rsync -r --delete'
f' --exclude=.git'
f' --exclude=.idea'
f" --exclude='*.pyc'"
f' --exclude=frontend/node_modules'
f' --exclude=backend/data/*.json'
f' --exclude=tests'
f' . {workdir}')
copy_insight_config_cmd = (
f' cp backend/data/insight.json {workdir}/backend/data/insight.json')
# copy_db_conf = "echo \'SQLALCHEMY_DATABASE_URI=\"{cloud_db_uri}\"\' > {workdir}/backends/instance/config.py".format(
# workdir=stage.workdir,
# cloud_db_uri=stage.cloud_db_uri)
copy_app_data = '\n'.join([
f'cat > {workdir}/backend/data/app.json <<EOL',
'{',
f' "notification_sender_email": "{notification_sender_email}",',
f' "app_title": "{app_title}"',
'}',
'EOL',
])
# We dont't use prod environment for the frontend to speed up deploy.
copy_prod_env = '\n'.join([
f'cat > {workdir}/frontend/src/environments/environment.ts <<EOL',
'export const environment = {',
' production: true,',
f' app_title: "{app_title}",',
f' enabled_stages: {enabled_stages}',
'}',
'EOL',
])
cmds = [
copy_src_cmd,
copy_insight_config_cmd,
copy_app_data,
copy_prod_env,
]
total = len(cmds)
for i, cmd in enumerate(cmds):
shared.execute_command(
f'Copy source code to working directory ({i + 1}/{total})',
cmd,
cwd=constants.PROJECT_DIR,
debug=debug)
def create_service_account_key_if_needed(stage, debug=False):
if shared.check_service_account_file(stage):
click.echo(" Service account key already exists.")
return
service_account_file = shared.get_service_account_file(stage)
gcloud_command = "$GOOGLE_CLOUD_SDK/bin/gcloud --quiet"
command = "{gcloud_bin} iam service-accounts keys create \"{service_account_file}\" \
--iam-account=\"{project_id}@appspot.gserviceaccount.com\" \
--key-file-type='json' \
--project={project_id}".format(
gcloud_bin=gcloud_command,
project_id=stage.project_id_gae,
service_account_file=service_account_file)
shared.execute_command("Create the service account key", command, debug=debug)