def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def get_git_shas_for_service(service, deploy_groups, soa_dir):
"""Returns a list of 2-tuples of the form (sha, timestamp) for each deploy tag in a service's git
repository"""
if service is None:
return []
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = {
config.get_deploy_group()
for config in get_instance_config_for_service(
service=service,
soa_dir=soa_dir,
)
}
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups,
deploy_groups)
previously_deployed_shas = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match['deploy_group']
tstamp = regex_match['tstamp']
except KeyError:
pass
else:
# note that all strings are greater than ''
if deploy_group in deploy_groups and tstamp > previously_deployed_shas.get(
sha, ''):
previously_deployed_shas[sha] = tstamp
return previously_deployed_shas.items()
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
if 'refs/heads/paasta-%s' % service_config.get_deploy_group(
) not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def get_branches(service):
paasta_branches = set(get_branches_for_service(SOA_DIR, service))
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
for branch in paasta_branches:
if 'refs/heads/%s' % branch in remote_refs:
yield branch
def get_git_shas_for_service(service, deploy_groups, soa_dir):
"""Returns a dictionary of 2-tuples of the form (timestamp, deploy_group) for each deploy sha"""
if service is None:
return []
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = list_deploy_groups(
service=service,
soa_dir=soa_dir,
)
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups, deploy_groups)
previously_deployed_shas = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match['deploy_group']
tstamp = regex_match['tstamp']
except KeyError:
pass
else:
# Now we filter and dedup by picking the most recent sha for a deploy group
# Note that all strings are greater than ''
if deploy_group in deploy_groups:
tstamp_so_far = previously_deployed_shas.get(sha, ('all', ''))[1]
if tstamp > tstamp_so_far:
previously_deployed_shas[sha] = (tstamp, deploy_group)
return previously_deployed_shas
def get_branches(service):
paasta_branches = set(get_branches_for_service(SOA_DIR, service))
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
for branch in paasta_branches:
if 'refs/heads/%s' % branch in remote_refs:
yield branch
def get_versions_for_service(
service: str, deploy_groups: Collection[str],
soa_dir: str) -> Mapping[DeploymentVersion, Tuple[str, str]]:
"""Returns a dictionary of 2-tuples of the form (timestamp, deploy_group) for each version tuple of (deploy sha, image_version)"""
if service is None:
return {}
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups,
deploy_groups)
previously_deployed_versions: Dict[DeploymentVersion, Tuple[str, str]] = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match["deploy_group"]
tstamp = regex_match["tstamp"]
image_version = regex_match["image_version"]
except KeyError:
pass
else:
# Now we filter and dedup by picking the most recent sha for a deploy group
# Note that all strings are greater than ''
if deploy_group in deploy_groups:
version = DeploymentVersion(sha=sha,
image_version=image_version)
tstamp_so_far = previously_deployed_versions.get(
version, ("all", ""))[1]
if tstamp > tstamp_so_far:
previously_deployed_versions[version] = (tstamp,
deploy_group)
return previously_deployed_versions
def paasta_wait_for_deployment(args):
"""Wrapping wait_for_deployment"""
if args.verbose:
log.setLevel(level=logging.DEBUG)
else:
log.setLevel(level=logging.INFO)
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
if args.git_url is None:
args.git_url = get_git_url(service=service, soa_dir=args.soa_dir)
try:
validate_full_git_sha(args.commit)
except ArgumentTypeError:
refs = remote_git.list_remote_refs(args.git_url)
commits = short_to_full_git_sha(short=args.commit, refs=refs)
if len(commits) != 1:
raise ValueError(
"%s matched %d git shas (with refs pointing at them). Must match exactly 1."
% (args.commit, len(commits)), )
args.commit = commits[0]
try:
validate_service_name(service, soa_dir=args.soa_dir)
validate_deploy_group(args.deploy_group, service, args.soa_dir)
validate_git_sha(
args.commit,
args.git_url,
args.deploy_group,
service,
)
except (GitShaError, DeployGroupError, NoSuchService) as e:
paasta_print(PaastaColors.red('{}'.format(e)))
return 1
try:
wait_for_deployment(
service=service,
deploy_group=args.deploy_group,
git_sha=args.commit,
soa_dir=args.soa_dir,
timeout=args.timeout,
)
_log(
service=service,
component='deploy',
line=("Deployment of {} for {} complete".format(
args.commit, args.deploy_group)),
level='event',
)
except (KeyboardInterrupt, TimeoutError, NoSuchCluster):
report_waiting_aborted(service, args.deploy_group)
return 1
return 0
def get_branches(service, soa_dir):
paasta_control_branches = set(('paasta-%s' % config.get_branch()
for config in get_instance_config_for_service(soa_dir, service)))
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
for branch in paasta_control_branches:
if 'refs/heads/%s' % branch in remote_refs:
yield branch
def get_deploy_group_mappings(soa_dir, service, old_mappings):
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new
mapping.
:returns: A dictionary mapping service:deploy_group to a dictionary containing:
- 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker
registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if
the other properties of this app have not changed.
"""
mappings = {}
service_configs = get_instance_config_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = dict(
(config.get_branch(), config.get_deploy_group())
for config in service_configs)
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return {}
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
deploy_ref_name = 'refs/heads/paasta-%s' % deploy_group
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = '%s:paasta-%s' % (service, control_branch)
deploy_group_branch_alias = '%s:paasta-%s' % (service,
deploy_group)
docker_image = build_docker_image_name(service, commit_sha)
log.info('Mapping deploy_group %s to docker image %s',
deploy_group_branch_alias, docker_image)
mapping = mappings.setdefault(control_branch_alias, {})
mapping['docker_image'] = docker_image
desired_state, force_bounce = get_desired_state(
service=service,
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
mapping['desired_state'] = desired_state
mapping['force_bounce'] = force_bounce
return mappings
def get_latest_marked_sha(git_url, deploy_group):
"""Return the latest marked for deployment git sha or ''"""
refs = list_remote_refs(git_url)
last_ref = ""
for ref in refs:
if (ref.startswith(f"refs/tags/paasta-{deploy_group}-")
and ref.endswith("-deploy") and ref > last_ref):
last_ref = ref
return refs[last_ref] if last_ref else ""
def get_latest_marked_sha(git_url, deploy_group):
"""Return the latest marked for deployment git sha or ''"""
refs = list_remote_refs(git_url)
last_ref = ''
for ref in refs:
if (ref.startswith('refs/tags/paasta-{}-'.format(deploy_group))
and ref.endswith('-deploy') and ref > last_ref):
last_ref = ref
return refs[last_ref] if last_ref else ''
def get_latest_marked_sha(git_url, deploy_group):
"""Return the latest marked for deployment git sha or ''"""
refs = list_remote_refs(git_url)
last_ref = ''
for ref in refs:
if (ref.startswith('refs/tags/paasta-{}-'.format(deploy_group)) and
ref.endswith('-deploy')) and ref > last_ref:
last_ref = ref
return refs[last_ref] if last_ref else ''
def get_latest_marked_version(
git_url: str, deploy_group: str) -> Optional[DeploymentVersion]:
"""Return the latest marked for deployment version or None"""
# TODO: correct this function for new tag format
refs = list_remote_refs(git_url)
_, sha, image_version = get_latest_deployment_tag(refs, deploy_group)
if sha:
return DeploymentVersion(sha=sha, image_version=image_version)
# We did not find a ref for this deploy group
return None
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
clusters = args.clusters
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
try:
remote_refs = remote_git.list_remote_refs(
utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print msg
return 1
invalid_deploy_groups = []
for cluster in clusters:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
clusters = args.clusters
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
for cluster in clusters:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def get_deploy_group_mappings(soa_dir, service, old_mappings):
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new
mapping.
:returns: A dictionary mapping service:deploy_group to a dictionary containing:
- 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker
registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if
the other properties of this app have not changed.
"""
mappings = {}
service_configs = get_instance_config_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = dict((config.get_branch(), config.get_deploy_group()) for config in service_configs)
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return {}
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
deploy_ref_name = 'refs/heads/paasta-%s' % deploy_group
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = '%s:paasta-%s' % (service, control_branch)
deploy_group_branch_alias = '%s:paasta-%s' % (service, deploy_group)
docker_image = build_docker_image_name(service, commit_sha)
log.info('Mapping deploy_group %s to docker image %s', deploy_group_branch_alias, docker_image)
mapping = mappings.setdefault(control_branch_alias, {})
mapping['docker_image'] = docker_image
desired_state, force_bounce = get_desired_state(
service=service,
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
mapping['desired_state'] = desired_state
mapping['force_bounce'] = force_bounce
return mappings
def validate_git_sha(sha, git_url):
try:
validate_full_git_sha(sha)
return sha
except argparse.ArgumentTypeError:
refs = remote_git.list_remote_refs(git_url)
commits = short_to_full_git_sha(short=sha, refs=refs)
if len(commits) != 1:
raise ValueError(
"%s matched %d git shas (with refs pointing at them). Must match exactly 1."
% (sha, len(commits)))
return commits[0]
def test_non_ascii_tags():
"""git tags can be UTF-8 encoded"""
with mock.patch(
"dulwich.client.get_transport_and_path",
autospec=True,
return_value=(
mock.Mock(
**{"fetch_pack.return_value": {"☃".encode("UTF-8"): b"deadbeef"}}
),
"path",
),
):
with mock.patch("time.sleep", autospec=True):
ret = remote_git.list_remote_refs("git-url")
assert ret == {"☃": "deadbeef"}
def test_non_ascii_tags():
"""git tags can be UTF-8 encoded"""
with mock.patch(
'dulwich.client.get_transport_and_path',
autospec=True,
return_value=(
mock.Mock(**{
'fetch_pack.return_value': {
'☃'.encode('UTF-8'): b'deadbeef'
}
}),
'path',
),
):
ret = remote_git.list_remote_refs('git-url')
assert ret == {'☃': 'deadbeef'}
def paasta_get_latest_deployment(args):
service = args.service
deploy_group = args.deploy_group
soa_dir = args.soa_dir
validate_service_name(service, soa_dir)
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = list_remote_refs(git_url)
_, git_sha = get_latest_deployment_tag(remote_refs, deploy_group)
if not git_sha:
print PaastaColors.red("A deployment could not be found for %s in %s" % (deploy_group, service))
return 1
else:
print git_sha
return 0
def paasta_get_latest_deployment(args):
service = args.service
deploy_group = args.deploy_group
soa_dir = args.soa_dir
validate_service_name(service, soa_dir)
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = list_remote_refs(git_url)
_, git_sha = get_latest_deployment_tag(remote_refs, deploy_group)
if not git_sha:
print PaastaColors.red("A deployment could not be found for %s in %s" %
(deploy_group, service))
return 1
else:
print git_sha
return 0
def get_branch_mappings(soa_dir, service, old_mappings):
"""Gets mappings from service:branch_name to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new
mapping.
:returns: A dictionary mapping service:branch_name to a dictionary containing:
- 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker
registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if
the other properties of this app have not changed.
"""
mappings = {}
valid_branches = get_branches_for_service(soa_dir, service)
if not valid_branches:
log.info("Service %s has no valid branches. Skipping.", service)
return {}
git_url = get_git_url(service, soa_dir=soa_dir)
remote_refs = remote_git.list_remote_refs(git_url)
for branch in valid_branches:
ref_name = "refs/heads/%s" % branch
if ref_name in remote_refs:
commit_sha = remote_refs[ref_name]
branch_alias = "%s:%s" % (service, branch)
docker_image = build_docker_image_name(service, commit_sha)
log.info("Mapping branch %s to docker image %s", branch_alias, docker_image)
mapping = mappings.setdefault(branch_alias, {})
mapping["docker_image"] = docker_image
desired_state, force_bounce = get_desired_state(service, branch, remote_refs)
mapping["desired_state"] = desired_state
mapping["force_bounce"] = force_bounce
return mappings
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
return 1
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def get_git_shas_for_service(service, deploy_groups, soa_dir):
"""Returns a dictionary of 2-tuples of the form (timestamp, deploy_group) for each deploy sha"""
if service is None:
return []
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = list_deploy_groups(
service=service,
soa_dir=soa_dir,
)
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups, deploy_groups)
previously_deployed_shas = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match['deploy_group']
tstamp = regex_match['tstamp']
except KeyError:
pass
else:
# note that all strings are greater than ''
if deploy_group in deploy_groups and tstamp > previously_deployed_shas.get(sha, ''):
previously_deployed_shas[sha] = (tstamp, deploy_group)
return previously_deployed_shas.items()
def get_git_shas_for_service(service, deploy_groups, soa_dir):
"""Returns a list of 2-tuples of the form (sha, timestamp) for each deploy tag in a service's git
repository"""
if service is None:
return []
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = {config.get_deploy_group() for config in get_instance_config_for_service(
service=service,
soa_dir=soa_dir,
)}
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups, deploy_groups)
previously_deployed_shas = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match['deploy_group']
tstamp = regex_match['tstamp']
except KeyError:
pass
else:
# note that all strings are greater than ''
if deploy_group in deploy_groups and tstamp > previously_deployed_shas.get(sha, ''):
previously_deployed_shas[sha] = tstamp
return previously_deployed_shas.items()
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
if args.instances is not None:
instances = args.instances.split(",")
else:
instances = None
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster in clusters:
# If they haven't specified what instances to act on, do it for all of them.
# If they have specified what instances, only iterate over them if they're
# actually within this cluster.
if instances is None:
cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
else:
all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
cluster_instances = all_cluster_instances.intersection(set(instances))
for instance in cluster_instances:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def get_remote_refs(service, soa_dir):
if service not in REMOTE_REFS:
REMOTE_REFS[service] = remote_git.list_remote_refs(
utils.get_git_url(service, soa_dir))
return REMOTE_REFS[service]
def get_deploy_group_mappings(
soa_dir: str,
service: str,
) -> Tuple[Dict[str, V1_Mapping], V2_Mappings]:
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary mapping service:deploy_group to a dictionary
containing:
- 'docker_image': something like "services-service:paasta-hash". This is
relative to the paasta docker registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch
should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this
value should trigger a bounce, even if the other properties of this app
have not changed.
"""
mappings: Dict[str, V1_Mapping] = {}
v2_mappings: V2_Mappings = {'deployments': {}, 'controls': {}}
service_configs = get_instance_configs_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = {
config.get_branch(): config.get_deploy_group()
for config in service_configs
}
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return mappings, v2_mappings
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
(deploy_ref_name,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = f'{service}:paasta-{control_branch}'
control_branch_alias_v2 = f'{service}:{control_branch}'
docker_image = build_docker_image_name(service, commit_sha)
desired_state, force_bounce = get_desired_state(
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
log.info('Mapping %s to docker image %s', control_branch,
docker_image)
v2_mappings['deployments'][deploy_group] = {
'docker_image': docker_image,
'git_sha': commit_sha,
}
mappings[control_branch_alias] = {
'docker_image': docker_image,
'desired_state': desired_state,
'force_bounce': force_bounce,
}
v2_mappings['controls'][control_branch_alias_v2] = {
'desired_state': desired_state,
'force_bounce': force_bounce,
}
return mappings, v2_mappings
def paasta_mark_for_deployment(args):
"""Wrapping mark_for_deployment"""
if args.verbose:
log.setLevel(level=logging.DEBUG)
else:
log.setLevel(level=logging.INFO)
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir=args.soa_dir)
in_use_deploy_groups = list_deploy_groups(
service=service,
soa_dir=args.soa_dir,
)
_, invalid_deploy_groups = validate_given_deploy_groups(
in_use_deploy_groups, [args.deploy_group])
if len(invalid_deploy_groups) == 1:
paasta_print(
PaastaColors.red(
"ERROR: These deploy groups are not currently used anywhere: %s.\n"
% (",").join(invalid_deploy_groups), ))
paasta_print(
PaastaColors.red(
"This isn't technically wrong because you can mark-for-deployment before deploying there",
))
paasta_print(
PaastaColors.red(
"but this is probably a typo. Did you mean one of these in-use deploy groups?:"
))
paasta_print(
PaastaColors.red(" %s" % (",").join(in_use_deploy_groups)))
paasta_print()
paasta_print(PaastaColors.red("Continuing regardless..."))
if args.git_url is None:
args.git_url = get_git_url(service=service, soa_dir=args.soa_dir)
try:
validate_full_git_sha(args.commit)
except ArgumentTypeError:
refs = remote_git.list_remote_refs(args.git_url)
commits = short_to_full_git_sha(short=args.commit, refs=refs)
if len(commits) != 1:
raise ValueError(
"%s matched %d git shas (with refs pointing at them). Must match exactly 1."
% (args.commit, len(commits)), )
args.commit = commits[0]
old_git_sha = get_currently_deployed_sha(service=service,
deploy_group=args.deploy_group)
if old_git_sha == args.commit:
paasta_print(
"Warning: The sha asked to be deployed already matches what is set to be deployed:"
)
paasta_print(old_git_sha)
paasta_print("Continuing anyway.")
if args.verify_image:
if not is_docker_image_already_in_registry(service, args.soa_dir,
args.commit):
raise ValueError(
'Failed to find image in the registry for the following sha %s'
% args.commit)
ret = mark_for_deployment(
git_url=args.git_url,
deploy_group=args.deploy_group,
service=service,
commit=args.commit,
)
if args.block and ret == 0:
try:
wait_for_deployment(
service=service,
deploy_group=args.deploy_group,
git_sha=args.commit,
soa_dir=args.soa_dir,
timeout=args.timeout,
)
line = "Deployment of {} for {} complete".format(
args.commit, args.deploy_group)
_log(
service=service,
component='deploy',
line=line,
level='event',
)
except (KeyboardInterrupt, TimeoutError):
if args.auto_rollback is True:
if old_git_sha == args.commit:
paasta_print(
"Error: --auto-rollback was requested, but the previous sha"
)
paasta_print(
"is the same that was requested with --commit. Can't rollback"
)
paasta_print("automatically.")
else:
paasta_print(
"Auto-Rollback requested. Marking the previous sha")
paasta_print("(%s) for %s as desired." %
(args.deploy_group, old_git_sha))
mark_for_deployment(
git_url=args.git_url,
deploy_group=args.deploy_group,
service=service,
commit=old_git_sha,
)
else:
report_waiting_aborted(service, args.deploy_group)
ret = 1
except NoSuchCluster:
report_waiting_aborted(service, args.deploy_group)
if old_git_sha is not None and old_git_sha != args.commit and not args.auto_rollback:
paasta_print()
paasta_print("If you wish to roll back, you can run:")
paasta_print()
paasta_print(
PaastaColors.bold(
" paasta rollback --service %s --deploy-group %s --commit %s "
% (
service,
args.deploy_group,
old_git_sha,
)), )
return ret
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {s for service_list in pargs.values() for s in service_list.keys()}
if len(affected_services) > 1:
paasta_print(PaastaColors.red("Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
paasta_print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
paasta_print(" Service %s:" % service)
paasta_print(" Instances %s" % ",".join(instances))
if sys.stdin.isatty():
confirm = choice.Binary('Are you sure you want to continue?', False).ask()
else:
confirm = False
if not confirm:
paasta_print()
paasta_print("exiting")
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
paasta_print(msg)
return 1
for instance in instances:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
paasta_print("No branches found for %s in %s." %
(", ".join(invalid_deploy_groups), remote_refs))
paasta_print("Has %s been deployed there yet?" % service)
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
instances = args.instances.split(",") if args.instances else None
# assert that each of the clusters that the user specifies are 'valid'
# for the instance list provided; that is, assert that at least one of the instances
# provided in the -i argument is deployed there.
# if there are no instances defined in the args, then assert that the service
# is deployed to that cluster.
# If args.clusters is falsey, then default to *all* clusters that a service is deployed to,
# and we figure out which ones are needed for each service later.
if instances:
instance_clusters = [list_clusters(service, soa_dir, instance) for instance in args.instances]
valid_clusters = sorted(list(set([cluster for cluster_list in instance_clusters for cluster in cluster_list])))
else:
valid_clusters = list_clusters(service, soa_dir)
if args.clusters:
clusters = args.clusters.split(",")
invalid_clusters = [cluster for cluster in clusters if cluster not in valid_clusters]
if invalid_clusters:
print ("Invalid cluster name(s) specified: %s." "Valid options: %s") % (
" ".join(invalid_clusters),
" ".join(valid_clusters),
)
return 1
else:
clusters = valid_clusters
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster in clusters:
# If they haven't specified what instances to act on, do it for all of them.
# If they have specified what instances, only iterate over them if they're
# actually within this cluster.
if instances is None:
cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
print ("no instances specified; restarting all instances for service")
else:
all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
cluster_instances = all_cluster_instances.intersection(set(instances))
for instance in cluster_instances:
service_config = get_instance_config(
service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config, force_bounce=force_bounce, desired_state=desired_state
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % (", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
instances = args.instances.split(",") if args.instances else None
# assert that each of the clusters that the user specifies are 'valid'
# for the instance list provided; that is, assert that at least one of the instances
# provided in the -i argument is deployed there.
# if there are no instances defined in the args, then assert that the service
# is deployed to that cluster.
# If args.clusters is falsey, then default to *all* clusters that a service is deployed to,
# and we figure out which ones are needed for each service later.
if instances:
instance_clusters = [
list_clusters(service, soa_dir, instance)
for instance in args.instances
]
valid_clusters = sorted(
list(
set([
cluster for cluster_list in instance_clusters
for cluster in cluster_list
])))
else:
valid_clusters = list_clusters(service, soa_dir)
if args.clusters:
clusters = args.clusters.split(",")
invalid_clusters = [
cluster for cluster in clusters if cluster not in valid_clusters
]
if invalid_clusters:
print("Invalid cluster name(s) specified: %s."
"Valid options: %s") % (" ".join(invalid_clusters),
" ".join(valid_clusters))
return 1
else:
clusters = valid_clusters
try:
remote_refs = remote_git.list_remote_refs(
utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print msg
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster in clusters:
# If they haven't specified what instances to act on, do it for all of them.
# If they have specified what instances, only iterate over them if they're
# actually within this cluster.
if instances is None:
cluster_instances = list_all_instances_for_service(
service, clusters=[cluster], soa_dir=soa_dir)
print(
"no instances specified; restarting all instances for service")
else:
all_cluster_instances = list_all_instances_for_service(
service, clusters=[cluster], soa_dir=soa_dir)
cluster_instances = all_cluster_instances.intersection(
set(instances))
for instance in cluster_instances:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(
datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig
) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(
service_config,
ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val