def reset(stage_name, debug):
"""Reset pipeline statuses."""
click.echo(click.style(">>>> Reset pipelines", fg='magenta', bold=True))
stage_name, stage = fetch_stage_or_default(stage_name, debug=debug)
if stage is None:
click.echo(click.style("Fix that issue by running: `$ crmint cloud setup`", fg='green'))
exit(1)
# Enriches stage with other variables.
stage = shared.before_hook(stage, stage_name)
# Runs setup stages.
components = [
install_required_packages,
display_workdir,
copy_src_to_workdir,
install_backends_dependencies,
download_cloud_sql_proxy,
start_cloud_sql_proxy,
prepare_flask_envars,
run_reset_pipelines,
stop_cloud_sql_proxy,
]
for component in components:
component(stage, debug=debug)
click.echo(click.style("Done.", fg='magenta', bold=True))
def _setup(stage_name, debug=False):
"""Setup the GCP environment for deploying CRMint."""
click.echo(click.style('>>>> Setup', fg='magenta', bold=True))
stage_name, stage = fetch_stage_or_default(stage_name, debug=debug)
if stage is None:
sys.exit(1)
# Enriches stage with other variables.
stage = shared.before_hook(stage, stage_name)
# Runs setup steps.
components = [
activate_services,
create_appengine,
grant_cloud_build_permissions,
grant_app_engine_default_service_account_permissions,
create_cloudsql_instance_if_needed,
create_cloudsql_user_if_needed,
create_cloudsql_database_if_needed,
create_pubsub_topics,
create_pubsub_subscriptions,
grant_pubsub_permissions,
create_scheduler_job,
#download_config_files,
]
for component in components:
component(stage, debug=debug)
click.echo(click.style('Done.', fg='magenta', bold=True))
def now(stage_name, debug):
"""Generate Vertex AI pipelines."""
msg = click.style(
" _ _ ___________ _____ _______ __ ___ _____ _ _ _____ _ _\n",
fg='bright_blue')
msg += click.style(
"| | | | ___| ___ \_ _| ___\ \ / / / _ \|_ _| | \ | | _ || | | |\n",
fg='bright_blue')
msg += click.style(
"| | | | |__ | |_/ / | | | |__ \ V / / /_\ \ | | | \| | | | || | | |\n",
fg='bright_red')
msg += click.style(
"| | | | __|| / | | | __| / \ | _ | | | | . ` | | | || |/\| |\n",
fg='bright_red')
msg += click.style(
"\ \_/ / |___| |\ \ | | | |___/ /^\ \ | | | |_| |_ | |\ \ \_/ /\ /\ /\n",
fg='bright_yellow')
msg += click.style(
" \___/\____/\_| \_| \_/ \____/\/ \/ \_| |_/\___/ \_| \_/\___/ \/ \/\n",
fg='bright_green')
click.echo(msg)
stage_name, stage = cloud.fetch_stage_or_default(stage_name,
debug=debug,
silent_step_name=True)
stage = shared.before_hook(stage, stage_name)
platforms = ['GA4', 'Universal Analytics']
click.echo(
'Vertex AI Now is available for both GA4 & Universal Analytics\n'
'Google Analytics property types.\n'
'--------------------------------------------')
for i, p in enumerate(platforms):
click.echo(f'{i + 1}) {p}')
ind = click.prompt(
'Enter the index for the Google Analytics property type', type=int) - 1
platform = platforms[ind]
if platform == 'GA4':
training_file, prediction_file = pipelines._get_ga4_config(stage,
ml='vertex')
if platform == 'Universal Analytics':
training_file, prediction_file = pipelines._get_ua_config(stage,
ml='vertex')
local_db_uri = stage.local_db_uri
env_vars = f'DATABASE_URI="{local_db_uri}" FLASK_APP=controller_app.py'
cloud.install_required_packages(stage)
cloud.display_workdir(stage)
cloud.copy_src_to_workdir(stage)
cloud.download_cloud_sql_proxy(stage)
cloud.start_cloud_sql_proxy(stage)
cloud.install_python_packages(stage)
cmd_workdir = os.path.join(stage.workdir, 'backend')
cmd = (' . .venv_controller/bin/activate &&'
f' {env_vars} python -m flask import-pipelines {training_file} &&'
f' {env_vars} python -m flask import-pipelines {prediction_file}')
shared.execute_command('Importing training & prediction pipelines',
cmd,
cwd=cmd_workdir,
debug=debug)
cloud.stop_cloud_sql_proxy(stage)
def _deploy(stage_name,
debug=False,
frontend=False,
controller=False,
jobs=False,
dispatch_rules=False,
db_migrations=False):
"""Deploy CRMint on GCP."""
click.echo(click.style('>>>> Deploy', fg='magenta', bold=True))
stage_name, stage = fetch_stage_or_default(stage_name, debug=debug)
if stage is None:
click.echo(
click.style('Fix that issue by running: $ crmint cloud setup',
fg='green'))
sys.exit(1)
# Enriches stage with other variables.
stage = shared.before_hook(stage, stage_name)
# If no specific components were specified for deploy, then deploy all.
if not (frontend or controller or jobs or dispatch_rules or db_migrations):
frontend = True
controller = True
jobs = True
dispatch_rules = True
db_migrations = True
# Runs deploy steps.
components = [
install_required_packages,
display_workdir,
copy_src_to_workdir,
]
if frontend:
components.append(deploy_frontend)
if controller:
components.append(deploy_controller)
if jobs:
components.append(deploy_jobs)
if dispatch_rules:
components.append(deploy_dispatch_rules)
if db_migrations:
components.extend([
download_cloud_sql_proxy,
start_cloud_sql_proxy,
install_python_packages,
run_db_migrations,
stop_cloud_sql_proxy,
display_appengine_hostname,
])
for component in components:
component(stage, debug=debug)
click.echo(click.style("Done.", fg='magenta', bold=True))
def deploy(stage_name, debug, skip_deploy_backends, skip_deploy_frontend):
"""Deploy CRMint on GCP."""
click.echo(click.style(">>>> Deploy", fg='magenta', bold=True))
stage_name, stage = fetch_stage_or_default(stage_name, debug=debug)
if stage is None:
click.echo(
click.style("Fix that issue by running: $ crmint cloud setup",
fg='green'))
exit(1)
# Enriches stage with other variables.
stage = shared.before_hook(stage, stage_name)
# Runs deploy steps.
components = [
install_required_packages,
display_workdir,
copy_src_to_workdir,
install_backends_dependencies,
deploy_frontend,
deploy_backends,
deploy_dispatch_rules,
download_cloud_sql_proxy,
start_cloud_sql_proxy,
prepare_flask_envars,
run_flask_db_upgrade,
run_flask_db_seeds,
stop_cloud_sql_proxy,
]
if skip_deploy_backends and (deploy_backends in components):
components.remove(deploy_backends)
if skip_deploy_frontend and (deploy_frontend in components):
components.remove(deploy_frontend)
for component in components:
component(stage, debug=debug)
click.echo(click.style("Done.", fg='magenta', bold=True))
def setup(stage_name, debug):
"""Setup the GCP environment for deploying CRMint."""
click.echo(click.style(">>>> Setup", fg='magenta', bold=True))
stage_name, stage = fetch_stage_or_default(stage_name, debug=debug)
if stage is None:
exit(1)
# Enriches stage with other variables.
stage = shared.before_hook(stage, stage_name)
# Runs setup steps.
components = [
create_appengine,
create_service_account_key_if_needed,
create_mysql_instance_if_needed,
create_mysql_user_if_needed,
create_mysql_database_if_needed,
activate_services,
download_config_files,
]
for component in components:
component(stage, debug=debug)
click.echo(click.style("Done.", fg='magenta', bold=True))