def build_gcloud_deploy_args(self, component, source_path):
config = component.descriptor['cloud_function']
env_vars = config.get('environment_variables', {})
env_vars_str = ("" if not env_vars else "--set-env-vars=" + ",".join(
["{}={}".format(key, value) for key, value in env_vars.items()]))
template_version = util.encode_version(component.code_version)
trigger_event = config['trigger_event'] \
if 'trigger_event' in config else 'providers/cloud.pubsub/eventTypes/topic.publish'
service_account = '--service-account {}'.format(config['service_account']) \
if 'service_account' in config else ''
vpc_connector = '--vpc-connector {}'.format(config['vpc_connector']) \
if 'vpc_connector' in config else ''
cmd = """
gcloud functions deploy {cloud_function_name} \
--runtime {runtime} \
--memory {memory} \
--source {source_path} \
--timeout {timeout} \
--update-labels='code_version={version}' \
--trigger-event {trigger_event} \
--trigger-resource {topic} \
--entry-point {entrypoint} \
--project {project_id} \
{retry} \
{service_account} \
{vpc_connector} \
{env_vars_str}
""".format(cloud_function_name=component.name,
runtime=config['runtime'],
timeout=config['timeout'],
memory=config['memory'],
trigger_event=trigger_event,
source_path=source_path,
version=util.encode_version(component.code_version),
retry="--retry" if config['retry'] else '',
service_account=service_account,
vpc_connector=vpc_connector,
entrypoint=config['entrypoint'],
topic=config['topic'],
project_id=self.project_id,
env_vars_str=env_vars_str).strip()
return self.prettify_output(cmd)
def build_gcloud_deploy_args(self, component, tmp_cloud_init_path, session, delete_template = False):
template_version = util.encode_version(component.code_version)
service_name = self.build_service_version_name(component)
gcloud_args = []
if delete_template and self._check_if_instance_template_exists(service_name, session):
gcloud_args.append(self._delete_instance_template(
component,
template_version,
service_name ))
gcloud_args.append(self._create_instance_template(
component,
tmp_cloud_init_path,
template_version,
service_name ))
instance_group = self._get_instance_group(component, session)
if instance_group:
gcloud_args.append(self._update_instance_group_if_needed(component, instance_group))
gcloud_args.append(self._rolling_update_instance_template(component, service_name, instance_group))
else:
gcloud_args.append(self._create_instance_group(component, service_name))
gcloud_args.append(self._attach_load_balancer_to_instance_group(component))
return [self.prettify_output(arg) for arg in gcloud_args]
def build_gcloud_deploy_args(self, component):
config = component.descriptor['cloud_run']
env_vars = config.get('environment_variables', {})
env_vars_str = ("" if not env_vars else "--set-env-vars=" + ",".join(
["{}={}".format(key, value) for key, value in env_vars.items()]))
base_args = """
gcloud beta run deploy {cloud_run_service_name} \
--memory={memory} \
--timeout={timeout} \
--update-labels='code_version={version}' \
--project={project_id} \
--region={region} \
--service-account={service_account} \
--platform={platform} \
--no-allow-unauthenticated \
{env_vars_str}
""".format(cloud_run_service_name=component.name,
timeout=config['timeout'],
memory=config['memory'],
version=util.encode_version(component.code_version),
service_account=config['service_account'],
platform=config['platform'],
region=config['region'],
project_id=self.project_id,
env_vars_str=env_vars_str).strip()
# Add image argument
extra_args = self.gcloud_extra_args(component)
cmd = base_args + " " + " ".join(extra_args)
return self.prettify_output(cmd)
def get_deployment_uri(self, component, versioned=False):
host_prefix = '{app_name}-dot-{project_id}'.format(
app_name = component.name,
project_id = self.project_id)
if versioned:
encoded_version = util.encode_version(component.code_version)
host_prefix = encoded_version + '-dot-' + host_prefix
return 'https://' + host_prefix + '-dot-us-central1.etsycloud.com'
def build_gcloud_deploy_args(self, component):
extra_args = self.gcloud_extra_args(component)
app_version = util.encode_version(component.code_version)
base_args = [
'gcloud', 'app', 'deploy', '--project', self.project_id,
'--version', app_version
]
return base_args + extra_args
def build_service_version_name(cls, component):
template_version = util.encode_version(component.code_version)
timestamp = int(time.time()*1000)
# The full instance group name may only be up to 61 characters long
# The timestamp and the hyphen before it are 14 characters, so the
# service name and version can be no longer than 47 characters
name_and_version = "{}-{}".format(component.name, template_version)
if len(name_and_version) > 47:
name_and_version_hash = sha1(name_and_version.encode("utf-8")).hexdigest()[:6]
name_and_version = name_and_version[:40] + "-" + name_and_version_hash
return "{}-{}".format(name_and_version, timestamp).lower()
def _delete_service_versions(self, service, versions):
subprocess.check_call([
'gcloud',
'app',
'versions',
'delete',
'--project',
self.project_id,
'--service',
service,
'--quiet',
] + [util.encode_version(v) for v in versions])
def _build_labels(self, component):
result = DataflowServiceVersion.build_labels(
component_name=component.name,
code_version=util.encode_version(component.code_version))
user_labels = component.descriptor['dataflow'].get('labels', {})
invalid_labels = [key for key in user_labels if key in result]
if invalid_labels:
raise Exception(
'Invalid use of reserved labels `{}`'.format(invalid_labels))
result.update(user_labels)
return result
def _deploy_with_rendered_app_yaml(self, component, rendered_app_yaml,
workdir, dry_run):
logging.info('Deploying app %s:%s', component.name,
util.encode_version(component.code_version))
with open(os.path.join(workdir, 'app.yaml'), 'w') as _out:
_out.write(rendered_app_yaml)
gcloud_args = self.build_gcloud_deploy_args(component)
if dry_run:
logging.info(
' dry run: Skipping execution of gcloud command `%s`',
' '.join(gcloud_args))
return
else:
subprocess.check_call(gcloud_args, cwd=workdir)
def get_deployment_uri(self, component, versioned=False):
host_prefix = '{app_name}-dot-{project_id}'.format(
app_name=self.build_app_name(component),
project_id=self.project_id)
if versioned:
encoded_version = util.encode_version(component.code_version)
host_prefix = encoded_version + '-dot-' + host_prefix
# I _think_ the maximum length for this is 63, but let's be a little conservative
if len(host_prefix) > 60:
# TODO: This is an extremely unlikely case, and may only be possible
# if versioned==True, but we could handle it by checking the GAE API
# to try to determine the proper hostname directly
raise Exception(('Host prefix {} is too long (length is {})'
', I think this will cause problems...').format(
len(host_prefix)))
return 'https://' + host_prefix + '.appspot.com'
def _build_job_name(self, component):
return component.name + '-' + util.encode_version(
component.code_version) + '-' + self._random_hash(4)