def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
output = []
try:
deployments = get_actual_deployments(service, soa_dir)
except NoDeploymentsAvailable:
deployments = {}
if deployments == {}:
output.append(" - N/A: Not deployed to any PaaSTA Clusters")
else:
service_config = load_service_namespace_config(service=service,
namespace="main",
soa_dir=soa_dir)
service_mode = service_config.get_mode()
for cluster in deployments_to_clusters(deployments):
if service_mode == "tcp":
service_port = service_config.get("proxy_port")
link = PaastaColors.cyan("%s://paasta-%s.yelp:%d/" %
(service_mode, cluster, service_port))
elif service_mode == "http" or service_mode == "https":
link = PaastaColors.cyan(
f"{service_mode}://{service}.paasta-{cluster}.yelp/")
else:
link = "N/A"
output.append(f" - {cluster} ({link})")
return output
def instance_status(request):
service = request.matchdict['service']
instance = request.matchdict['instance']
verbose = request.matchdict.get('verbose', False)
instance_status = {}
instance_status['service'] = service
instance_status['instance'] = instance
actual_deployments = get_actual_deployments(service, settings.soa_dir)
version = get_deployment_version(actual_deployments, settings.cluster, instance)
# exit if the deployment key is not found
if not version:
error_message = 'deployment key %s not found' % '.'.join([settings.cluster, instance])
raise InstanceFailure(error_message, 404)
instance_status['git_sha'] = version
try:
instance_type = validate_service_instance(service, instance, settings.cluster, settings.soa_dir)
if instance_type == 'marathon':
marathon_instance_status(instance_status, service, instance, verbose)
elif instance_type == 'chronos':
chronos_instance_status(instance_status, service, instance, verbose)
else:
error_message = 'Unknown instance_type %s of %s.%s' % (instance_type, service, instance)
raise InstanceFailure(error_message, 404)
except:
error_message = traceback.format_exc()
raise InstanceFailure(error_message, 500)
return instance_status
def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
output = []
try:
deployments = get_actual_deployments(service, soa_dir)
except NoDeploymentsAvailable:
deployments = {}
if deployments == {}:
output.append(' - N/A: Not deployed to any PaaSTA Clusters')
else:
service_config = load_service_namespace_config(
service=service,
namespace='main',
soa_dir=soa_dir,
)
service_mode = service_config.get_mode()
for cluster in deployments_to_clusters(deployments):
if service_mode == "tcp":
service_port = service_config.get('proxy_port')
link = PaastaColors.cyan('%s://paasta-%s.yelp:%d/' %
(service_mode, cluster, service_port))
elif service_mode == "http" or service_mode == "https":
link = PaastaColors.cyan('%s://%s.paasta-%s.yelp/' %
(service_mode, service, cluster))
else:
link = "N/A"
output.append(' - %s (%s)' % (cluster, link))
return output
def instance_status(request):
service = request.swagger_data.get('service')
instance = request.swagger_data.get('instance')
verbose = request.swagger_data.get('verbose', False)
instance_status: Dict[str, Any] = {}
instance_status['service'] = service
instance_status['instance'] = instance
try:
instance_type = validate_service_instance(service, instance, settings.cluster, settings.soa_dir)
except NoConfigurationForServiceError:
error_message = 'deployment key %s not found' % '.'.join([settings.cluster, instance])
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
if instance_type != 'flinkcluster':
try:
actual_deployments = get_actual_deployments(service, settings.soa_dir)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
version = get_deployment_version(actual_deployments, settings.cluster, instance)
# exit if the deployment key is not found
if not version:
error_message = 'deployment key %s not found' % '.'.join([settings.cluster, instance])
raise ApiFailure(error_message, 404)
instance_status['git_sha'] = version
else:
instance_status['git_sha'] = ''
try:
if instance_type == 'marathon':
instance_status['marathon'] = marathon_instance_status(instance_status, service, instance, verbose)
elif instance_type == 'chronos':
instance_status['chronos'] = chronos_instance_status(instance_status, service, instance, verbose)
elif instance_type == 'adhoc':
instance_status['adhoc'] = adhoc_instance_status(instance_status, service, instance, verbose)
elif instance_type == 'kubernetes':
instance_status['kubernetes'] = kubernetes_instance_status(instance_status, service, instance, verbose)
elif instance_type == 'flinkcluster':
status = flinkcluster_instance_status(instance_status, service, instance, verbose)
if status is not None:
instance_status['flinkcluster'] = {'status': status}
else:
instance_status['flinkcluster'] = {}
else:
error_message = f'Unknown instance_type {instance_type} of {service}.{instance}'
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
return instance_status
def test_get_actual_deployments(mock_get_deployments,):
mock_get_deployments.return_value = utils.DeploymentsJson(
{
"fake_service:paasta-b_cluster.b_instance": {"docker_image": "this_is_a_sha"},
"fake_service:paasta-a_cluster.a_instance": {"docker_image": "this_is_a_sha"},
}
)
expected = {"a_cluster.a_instance": "this_is_a_sha", "b_cluster.b_instance": "this_is_a_sha"}
actual = status.get_actual_deployments("fake_service")
assert expected == actual
def test_get_actual_deployments(mock_get_deployments, ):
mock_get_deployments.return_value = utils.DeploymentsJson({
'fake_service:paasta-b_cluster.b_instance': {
'docker_image': 'this_is_a_sha',
},
'fake_service:paasta-a_cluster.a_instance': {
'docker_image': 'this_is_a_sha',
}
})
expected = {
'a_cluster.a_instance': 'this_is_a_sha',
'b_cluster.b_instance': 'this_is_a_sha',
}
actual = status.get_actual_deployments('fake_service', '/fake/soa/dir')
assert expected == actual
def test_get_actual_deployments(mock_get_deployments,):
mock_get_deployments.return_value = utils.DeploymentsJson({
'fake_service:paasta-b_cluster.b_instance': {
'docker_image': 'this_is_a_sha',
},
'fake_service:paasta-a_cluster.a_instance': {
'docker_image': 'this_is_a_sha',
}
})
expected = {
'a_cluster.a_instance': 'this_is_a_sha',
'b_cluster.b_instance': 'this_is_a_sha',
}
actual = status.get_actual_deployments('fake_service', '/fake/soa/dir')
assert expected == actual
def instance_status(request):
service = request.swagger_data.get('service')
instance = request.swagger_data.get('instance')
verbose = request.matchdict.get('verbose', False)
instance_status = {}
instance_status['service'] = service
instance_status['instance'] = instance
try:
actual_deployments = get_actual_deployments(service, settings.soa_dir)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
version = get_deployment_version(actual_deployments, settings.cluster,
instance)
# exit if the deployment key is not found
if not version:
error_message = 'deployment key %s not found' % '.'.join(
[settings.cluster, instance])
raise ApiFailure(error_message, 404)
instance_status['git_sha'] = version
try:
instance_type = validate_service_instance(service, instance,
settings.cluster,
settings.soa_dir)
if instance_type == 'marathon':
instance_status['marathon'] = marathon_instance_status(
instance_status, service, instance, verbose)
elif instance_type == 'chronos':
instance_status['chronos'] = chronos_instance_status(
instance_status, service, instance, verbose)
elif instance_type == 'adhoc':
instance_status['adhoc'] = adhoc_instance_status(
instance_status, service, instance, verbose)
else:
error_message = 'Unknown instance_type %s of %s.%s' % (
instance_type, service, instance)
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
return instance_status
def get_deployments_strings(service, soa_dir):
output = []
try:
deployments = get_actual_deployments(service, soa_dir)
except NoDeploymentsAvailable:
deployments = {}
if deployments == {}:
output.append(' - N/A: Not deployed to any PaaSTA Clusters')
else:
service_config = load_service_namespace_config(service, 'main', soa_dir)
service_mode = service_config.get_mode()
for cluster in deployments_to_clusters(deployments):
if service_mode == "tcp":
service_port = service_config.get('proxy_port')
link = PaastaColors.cyan('%s://paasta-%s.yelp:%d/' % (service_mode, cluster, service_port))
elif service_mode == "http":
link = PaastaColors.cyan('%s://%s.paasta-%s.yelp/' % (service_mode, service, cluster))
else:
link = "N/A"
output.append(' - %s (%s)' % (cluster, link))
return output
def main() -> None:
args = parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
instances = []
return_codes = []
command = args.command
if (args.service_instance):
service_instance = args.service_instance
service, instance, _, __ = decompose_job_id(service_instance)
instances.append(instance)
elif (args.service and args.instances):
service = args.service
instances = args.instances.split(',')
else:
log.error(
"The name of service or the name of instance to inspect is missing. Exiting."
)
sys.exit(1)
# Setting up transparent cache for http API calls
requests_cache.install_cache("paasta_serviceinit", backend="memory")
cluster = load_system_paasta_config().get_cluster()
actual_deployments = get_actual_deployments(service, args.soa_dir)
clients = PaastaClients(cached=(command == 'status'))
instance_types = ['marathon', 'chronos', 'paasta_native', 'adhoc']
instance_types_map: Dict[str,
List[str]] = {it: []
for it in instance_types}
for instance in instances:
try:
instance_type = validate_service_instance(
service,
instance,
cluster,
args.soa_dir,
)
except Exception:
log.error(
('Exception raised while looking at service %s instance %s:'
).format(service, instance), )
log.error(traceback.format_exc())
return_codes.append(1)
continue
if instance_type not in instance_types:
log.error(
("I calculated an instance_type of {} for {} which I don't "
"know how to handle.").format(
instance_type,
compose_job_id(service, instance),
), )
return_codes.append(1)
else:
instance_types_map[instance_type].append(instance)
remote_run_frameworks = None
if len(instance_types_map['adhoc']) > 0:
remote_run_frameworks = paasta_remote_run.remote_run_frameworks()
service_config_loader = PaastaServiceConfigLoader(service)
for instance_type in instance_types:
if instance_type == 'marathon':
job_configs = {
jc.instance: jc
for jc in service_config_loader.instance_configs(
cluster=cluster,
instance_type_class=marathon_tools.MarathonServiceConfig,
)
}
for instance in instance_types_map[instance_type]:
try:
version = get_deployment_version(
actual_deployments,
cluster,
instance,
)
paasta_print('instance: %s' % PaastaColors.blue(instance))
paasta_print('Git sha: %s (desired)' % version)
if instance_type == 'marathon':
return_code = marathon_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
app_id=args.app_id,
clients=clients.marathon(),
job_config=job_configs[instance],
)
elif instance_type == 'chronos':
return_code = chronos_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
client=clients.chronos(),
)
elif instance_type == 'paasta_native':
return_code = paasta_native_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
)
elif instance_type == 'adhoc':
if command != 'status':
raise NotImplementedError
paasta_remote_run.remote_run_list_report(
service=service,
instance=instance,
cluster=cluster,
frameworks=remote_run_frameworks,
)
return_code = 0
except Exception:
log.error(('Exception raised while looking at service {} '
'instance {}:').format(service, instance), )
log.error(traceback.format_exc())
return_code = 1
return_codes.append(return_code)
sys.exit(max(return_codes))
def main():
args = parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
instances = []
return_codes = []
command = args.command
if (args.service_instance):
service_instance = args.service_instance
service, instance, _, __ = decompose_job_id(service_instance)
instances.append(instance)
elif (args.service and args.instances):
service = args.service
instances = args.instances.split(',')
else:
log.error(
"The name of service or the name of instance to inspect is missing. Exiting."
)
sys.exit(1)
# Setting up transparent cache for http API calls
requests_cache.install_cache("paasta_serviceinit", backend="memory")
cluster = load_system_paasta_config().get_cluster()
actual_deployments = get_actual_deployments(service, args.soa_dir)
clients = PaastaClients(cached=(command == 'status'))
for instance in instances:
try:
instance_type = validate_service_instance(service, instance,
cluster, args.soa_dir)
if instance_type == 'adhoc':
continue
version = get_deployment_version(actual_deployments, cluster,
instance)
paasta_print('instance: %s' % PaastaColors.blue(instance))
paasta_print('Git sha: %s (desired)' % version)
if instance_type == 'marathon':
return_code = marathon_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
app_id=args.app_id,
delta=args.delta,
client=clients.marathon(),
)
elif instance_type == 'chronos':
return_code = chronos_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
client=clients.chronos(),
)
elif instance_type == 'paasta_native':
return_code = paasta_native_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
)
else:
log.error(
"I calculated an instance_type of %s for %s which I don't know how to handle."
% (instance_type, compose_job_id(service, instance)))
return_code = 1
except Exception:
log.error(
'Exception raised while looking at service %s instance %s:' %
(service, instance))
log.error(traceback.format_exc())
return_code = 1
return_codes.append(return_code)
sys.exit(max(return_codes))
def main():
args = parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
instances = []
return_codes = []
command = args.command
if (args.service_instance):
service_instance = args.service_instance
service, instance, _, __ = decompose_job_id(service_instance)
instances.append(instance)
elif (args.service and args.instances):
service = args.service
instances = args.instances.split(',')
else:
log.error("The name of service or the name of instance to inspect is missing. Exiting.")
sys.exit(1)
# Setting up transparent cache for http API calls
requests_cache.install_cache("paasta_serviceinit", backend="memory")
cluster = load_system_paasta_config().get_cluster()
actual_deployments = get_actual_deployments(service, args.soa_dir)
for instance in instances:
# For an instance, there might be multiple versions running, e.g. in crossover bouncing.
# In addition, mesos master does not have information of a chronos service's git hash.
# The git sha in deployment.json is simply used here.
version = actual_deployments['.'.join((cluster, instance))][:8]
print 'instance: %s' % PaastaColors.blue(instance)
print 'Git sha: %s (desired)' % version
try:
instance_type = validate_service_instance(service, instance, cluster, args.soa_dir)
if instance_type == 'marathon':
return_code = marathon_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
app_id=args.app_id,
delta=args.delta,
)
elif instance_type == 'chronos':
return_code = chronos_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
)
else:
log.error("I calculated an instance_type of %s for %s which I don't know how to handle."
% (instance_type, compose_job_id(service, instance)))
return_code = 1
except:
log.error('Exception raised while looking at service %s instance %s:' % (service, instance))
log.error(traceback.format_exc())
return_code = 1
return_codes.append(return_code)
sys.exit(max(return_codes))
def paasta_rerun(args):
"""Reruns a Chronos job.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir) # exit with an error if the service doesn't exist
if args.execution_date:
execution_date = args.execution_date
else:
execution_date = None
all_clusters = list_clusters(soa_dir=soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir) # cluster.instance: sha
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir)) # cluster.instance
deployed_clusters = list_deployed_clusters(deploy_pipeline, actual_deployments)
deployed_cluster_instance = _get_cluster_instance(actual_deployments.keys())
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = deployed_clusters
for cluster in clusters:
print "cluster: %s" % cluster
if cluster not in all_clusters:
print " Warning: \"%s\" does not look like a valid cluster." % cluster
continue
if cluster not in deployed_clusters:
print " Warning: service \"%s\" has not been deployed to \"%s\" yet." % (service, cluster)
continue
if not deployed_cluster_instance[cluster].get(args.instance, False):
print (" Warning: instance \"%s\" is either invalid "
"or has not been deployed to \"%s\" yet." % (args.instance, cluster))
continue
try:
chronos_job_config = chronos_tools.load_chronos_job_config(
service, args.instance, cluster, load_deployments=False, soa_dir=soa_dir)
if chronos_tools.uses_time_variables(chronos_job_config) and execution_date is None:
print (" Warning: \"%s\" uses time variables interpolation, "
"please supply a `--execution_date` argument." % args.instance)
continue
except chronos_tools.UnknownChronosJobError as e:
print " Warning: %s" % e.message
continue
if execution_date is None:
execution_date = _get_default_execution_date()
rc, output = execute_chronos_rerun_on_remote_master(
service=service,
instancename=args.instance,
cluster=cluster,
verbose=args.verbose,
execution_date=execution_date.strftime(chronos_tools.EXECUTION_DATE_FORMAT)
)
if rc == 0:
print PaastaColors.green(' successfully created job')
else:
print PaastaColors.red(' error')
print output
def instance_status(request):
service = request.swagger_data.get("service")
instance = request.swagger_data.get("instance")
verbose = request.swagger_data.get("verbose") or 0
use_new = request.swagger_data.get("new") or False
include_smartstack = request.swagger_data.get("include_smartstack")
if include_smartstack is None:
include_smartstack = True
include_envoy = request.swagger_data.get("include_envoy")
if include_envoy is None:
include_envoy = True
include_mesos = request.swagger_data.get("include_mesos")
if include_mesos is None:
include_mesos = True
instance_status: Dict[str, Any] = {}
instance_status["service"] = service
instance_status["instance"] = instance
try:
instance_type = validate_service_instance(service, instance,
settings.cluster,
settings.soa_dir)
except NoConfigurationForServiceError:
error_message = no_configuration_for_service_message(
settings.cluster,
service,
instance,
)
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
if instance_type != "tron":
try:
actual_deployments = get_actual_deployments(
service, settings.soa_dir)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
version = get_deployment_version(actual_deployments, settings.cluster,
instance)
# exit if the deployment key is not found
if not version:
error_message = (
"Deployment key %s not found. Try to execute the corresponding pipeline if it's a fresh instance"
% ".".join([settings.cluster, instance]))
raise ApiFailure(error_message, 404)
instance_status["git_sha"] = version
else:
instance_status["git_sha"] = ""
try:
if instance_type == "marathon":
instance_status["marathon"] = marathon_instance_status(
instance_status,
service,
instance,
verbose,
include_smartstack=include_smartstack,
include_envoy=include_envoy,
include_mesos=include_mesos,
)
elif instance_type == "adhoc":
instance_status["adhoc"] = adhoc_instance_status(
instance_status, service, instance, verbose)
elif pik.can_handle(instance_type):
instance_status.update(
pik.instance_status(
service=service,
instance=instance,
verbose=verbose,
include_smartstack=include_smartstack,
include_envoy=include_envoy,
use_new=use_new,
instance_type=instance_type,
settings=settings,
))
elif instance_type == "tron":
instance_status["tron"] = tron_instance_status(
instance_status, service, instance, verbose)
else:
error_message = (
f"Unknown instance_type {instance_type} of {service}.{instance}"
)
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
return instance_status
def instance_status(request):
service = request.swagger_data.get("service")
instance = request.swagger_data.get("instance")
verbose = request.swagger_data.get("verbose") or 0
include_smartstack = request.swagger_data.get("include_smartstack")
if include_smartstack is None:
include_smartstack = True
include_mesos = request.swagger_data.get("include_mesos")
if include_mesos is None:
include_mesos = True
instance_status: Dict[str, Any] = {}
instance_status["service"] = service
instance_status["instance"] = instance
try:
instance_type = validate_service_instance(
service, instance, settings.cluster, settings.soa_dir
)
except NoConfigurationForServiceError:
error_message = (
"Deployment key %s not found. Try to execute the corresponding pipeline if it's a fresh instance"
% ".".join([settings.cluster, instance])
)
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
if instance_type != "tron":
try:
actual_deployments = get_actual_deployments(service, settings.soa_dir)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
version = get_deployment_version(actual_deployments, settings.cluster, instance)
# exit if the deployment key is not found
if not version:
error_message = (
"Deployment key %s not found. Try to execute the corresponding pipeline if it's a fresh instance"
% ".".join([settings.cluster, instance])
)
raise ApiFailure(error_message, 404)
instance_status["git_sha"] = version
else:
instance_status["git_sha"] = ""
try:
if instance_type == "marathon":
instance_status["marathon"] = marathon_instance_status(
instance_status,
service,
instance,
verbose,
include_smartstack=include_smartstack,
include_mesos=include_mesos,
)
elif instance_type == "adhoc":
instance_status["adhoc"] = adhoc_instance_status(
instance_status, service, instance, verbose
)
elif instance_type == "kubernetes":
instance_status["kubernetes"] = kubernetes_instance_status(
instance_status,
service,
instance,
verbose,
include_smartstack=include_smartstack,
instance_type=instance_type,
)
elif instance_type == "tron":
instance_status["tron"] = tron_instance_status(
instance_status, service, instance, verbose
)
elif instance_type in INSTANCE_TYPES_K8S:
cr_id_fn = cr_id_fn_for_instance_type(instance_type)
cr_id = cr_id_fn(service, instance)
status = kubernetes_cr_status(cr_id, verbose)
metadata = kubernetes_cr_metadata(cr_id, verbose)
instance_status[instance_type] = {}
if status is not None:
instance_status[instance_type]["status"] = status
if metadata is not None:
instance_status[instance_type]["metadata"] = metadata
else:
error_message = (
f"Unknown instance_type {instance_type} of {service}.{instance}"
)
raise ApiFailure(error_message, 404)
if instance_type == "cassandracluster":
instance_status["kubernetes"] = kubernetes_instance_status(
instance_status,
service,
instance,
verbose,
include_smartstack=include_smartstack,
instance_type=instance_type,
)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
return instance_status
def paasta_rerun(args):
"""Reruns a Chronos job.
:param args: argparse.Namespace obj created from sys.args by cli"""
system_paasta_config = load_system_paasta_config()
soa_dir = args.soa_dir
service = figure_out_service_name(
args, soa_dir) # exit with an error if the service doesn't exist
if args.execution_date:
execution_date = args.execution_date
else:
execution_date = None
all_clusters = list_clusters(soa_dir=soa_dir)
actual_deployments = get_actual_deployments(
service, soa_dir) # cluster.instance: sha
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(
service, soa_dir)) # cluster.instance
deployed_clusters = list_deployed_clusters(deploy_pipeline,
actual_deployments)
deployed_cluster_instance = _get_cluster_instance(
actual_deployments.keys())
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = deployed_clusters
for cluster in clusters:
print "cluster: %s" % cluster
if cluster not in all_clusters:
print " Warning: \"%s\" does not look like a valid cluster." % cluster
continue
if cluster not in deployed_clusters:
print " Warning: service \"%s\" has not been deployed to \"%s\" yet." % (
service, cluster)
continue
if not deployed_cluster_instance[cluster].get(args.instance, False):
print(
" Warning: instance \"%s\" is either invalid "
"or has not been deployed to \"%s\" yet." %
(args.instance, cluster))
continue
try:
chronos_job_config = chronos_tools.load_chronos_job_config(
service,
args.instance,
cluster,
load_deployments=False,
soa_dir=soa_dir)
if chronos_tools.uses_time_variables(
chronos_job_config) and execution_date is None:
print(
" Warning: \"%s\" uses time variables interpolation, "
"please supply a `--execution_date` argument." %
args.instance)
continue
except chronos_tools.UnknownChronosJobError as e:
print " Warning: %s" % e.message
continue
if execution_date is None:
execution_date = _get_default_execution_date()
rc, output = execute_chronos_rerun_on_remote_master(
service=service,
instancename=args.instance,
cluster=cluster,
verbose=args.verbose,
execution_date=execution_date.strftime(
chronos_tools.EXECUTION_DATE_FORMAT),
system_paasta_config=system_paasta_config,
)
if rc == 0:
print PaastaColors.green(' successfully created job')
else:
print PaastaColors.red(' error')
print output
def paasta_rerun(args):
"""Reruns a Chronos job.
:param args: argparse.Namespace obj created from sys.args by cli"""
system_paasta_config = load_system_paasta_config()
soa_dir = args.soa_dir
service = figure_out_service_name(
args, soa_dir) # exit with an error if the service doesn't exist
if args.execution_date:
execution_date = args.execution_date
else:
execution_date = None
all_clusters = list_clusters(soa_dir=soa_dir)
actual_deployments = get_actual_deployments(
service, soa_dir) # cluster.instance: sha
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(
service, soa_dir)) # cluster.instance
deployed_clusters = list_deployed_clusters(deploy_pipeline,
actual_deployments)
deployed_cluster_instance = _get_cluster_instance(
actual_deployments.keys())
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = deployed_clusters
for cluster in clusters:
paasta_print("cluster: %s" % cluster)
if cluster not in all_clusters:
paasta_print(
" Warning: \"%s\" does not look like a valid cluster." %
cluster)
continue
if cluster not in deployed_clusters:
paasta_print(
f" Warning: service \"{service}\" has not been deployed to \"{cluster}\" yet."
)
continue
if not deployed_cluster_instance[cluster].get(args.instance, False):
paasta_print((" Warning: instance \"%s\" is either invalid "
"or has not been deployed to \"%s\" yet." %
(args.instance, cluster)))
continue
try:
chronos_job_config = chronos_tools.load_chronos_job_config(
service,
args.instance,
cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if chronos_tools.uses_time_variables(
chronos_job_config) and execution_date is None:
paasta_print(
(" Warning: \"%s\" uses time variables interpolation, "
"please supply a `--execution_date` argument." %
args.instance))
continue
except NoConfigurationForServiceError as e:
paasta_print(" Warning: %s" % e)
continue
if execution_date is None:
execution_date = _get_default_execution_date()
related_job_configs = get_related_jobs_configs(cluster, service,
args.instance)
if not args.rerun_type and len(related_job_configs) > 1:
instance_names = sorted([
f'- {srv}{chronos_tools.INTERNAL_SPACER}{inst}'
for srv, inst in related_job_configs
if srv != service or inst != args.instance
])
paasta_print(PaastaColors.red(' error'))
paasta_print(
'Instance {instance} has dependency relations with the following jobs:\n'
'{relations}\n'
'\n'
'Please specify the rerun policy via --rerun-type argument'.
format(
instance=args.instance,
relations='\n'.join(instance_names),
), )
return
rc, output = execute_chronos_rerun_on_remote_master(
service=service,
instancename=args.instance,
cluster=cluster,
verbose=args.verbose,
execution_date=execution_date.strftime(
chronos_tools.EXECUTION_DATE_FORMAT),
system_paasta_config=system_paasta_config,
run_all_related_jobs=args.rerun_type == 'graph',
force_disabled=args.force_disabled,
)
if rc == 0:
paasta_print(PaastaColors.green(' successfully created job'))
else:
paasta_print(PaastaColors.red(' error'))
paasta_print(output)
def main():
args = parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
instances = []
return_codes = []
command = args.command
if (args.service_instance):
service_instance = args.service_instance
service, instance, _, __ = decompose_job_id(service_instance)
instances.append(instance)
elif (args.service and args.instances):
service = args.service
instances = args.instances.split(',')
else:
log.error(
"The name of service or the name of instance to inspect is missing. Exiting."
)
sys.exit(1)
# Setting up transparent cache for http API calls
requests_cache.install_cache("paasta_serviceinit", backend="memory")
cluster = load_system_paasta_config().get_cluster()
actual_deployments = get_actual_deployments(service, args.soa_dir)
for instance in instances:
# For an instance, there might be multiple versions running, e.g. in crossover bouncing.
# In addition, mesos master does not have information of a chronos service's git hash.
# The git sha in deployment.json is simply used here.
version = actual_deployments['.'.join((cluster, instance))][:8]
print 'instance: %s' % PaastaColors.blue(instance)
print 'Git sha: %s (desired)' % version
try:
instance_type = validate_service_instance(service, instance,
cluster, args.soa_dir)
if instance_type == 'marathon':
return_code = marathon_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
app_id=args.app_id,
delta=args.delta,
)
elif instance_type == 'chronos':
return_code = chronos_serviceinit.perform_command(
command=command,
service=service,
instance=instance,
cluster=cluster,
verbose=args.verbose,
soa_dir=args.soa_dir,
)
else:
log.error(
"I calculated an instance_type of %s for %s which I don't know how to handle."
% (instance_type, compose_job_id(service, instance)))
return_code = 1
except Exception:
log.error(
'Exception raised while looking at service %s instance %s:' %
(service, instance))
log.error(traceback.format_exc())
return_code = 1
return_codes.append(return_code)
sys.exit(max(return_codes))
def instance_status(request):
service = request.swagger_data.get("service")
instance = request.swagger_data.get("instance")
verbose = request.swagger_data.get("verbose") or 0
omit_smartstack = request.swagger_data.get("omit_smartstack") or False
omit_mesos = request.swagger_data.get("omit_mesos") or False
instance_status: Dict[str, Any] = {}
instance_status["service"] = service
instance_status["instance"] = instance
try:
instance_type = validate_service_instance(service, instance,
settings.cluster,
settings.soa_dir)
except NoConfigurationForServiceError:
error_message = "deployment key %s not found" % ".".join(
[settings.cluster, instance])
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
if instance_type != "flink" and instance_type != "tron":
try:
actual_deployments = get_actual_deployments(
service, settings.soa_dir)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
version = get_deployment_version(actual_deployments, settings.cluster,
instance)
# exit if the deployment key is not found
if not version:
error_message = "deployment key %s not found" % ".".join(
[settings.cluster, instance])
raise ApiFailure(error_message, 404)
instance_status["git_sha"] = version
else:
instance_status["git_sha"] = ""
try:
if instance_type == "marathon":
instance_status["marathon"] = marathon_instance_status(
instance_status,
service,
instance,
verbose,
omit_smartstack=omit_smartstack,
omit_mesos=omit_mesos,
)
elif instance_type == "chronos":
if verbose:
instance_status["chronos"] = chronos_instance_status(
service, instance, 1)
else:
instance_status["chronos"] = chronos_instance_status(
service, instance, 0)
elif instance_type == "adhoc":
instance_status["adhoc"] = adhoc_instance_status(
instance_status, service, instance, verbose)
elif instance_type == "kubernetes":
instance_status["kubernetes"] = kubernetes_instance_status(
instance_status, service, instance, verbose)
elif instance_type == "tron":
instance_status["tron"] = tron_instance_status(
instance_status, service, instance, verbose)
elif instance_type == "flink":
status = flink_instance_status(instance_status, service, instance,
verbose)
if status is not None:
instance_status["flink"] = {"status": status}
else:
instance_status["flink"] = {}
else:
error_message = (
f"Unknown instance_type {instance_type} of {service}.{instance}"
)
raise ApiFailure(error_message, 404)
except Exception:
error_message = traceback.format_exc()
raise ApiFailure(error_message, 500)
return instance_status
