def assemble():
"""Creates a build/$version directory with the artifacts to bake into the docker image"""
v = get_version(env)
prof = cfg.profile(env)
files = env.PROJECT_FILES
build = cfg.build_dir(v)
run("mkdir -p %s" % build)
for f in files:
expanded_path = f % env
recurse = "-r " if path.isdir(expanded_path) else " "
run("cp %s%s %s" % (recurse, expanded_path, build))
def get_version():
"""Computes the response of the '/version' endpoint.
Returns:
The value of the docker.get_version() or an error message.
"""
gs = app.context_graph_global_state
try:
version = docker.get_version(gs)
return flask.jsonify(utilities.make_response(version, 'version'))
except collector_error.CollectorError as e:
return flask.jsonify(utilities.make_error(str(e)))
except:
msg = ('get_version() failed with exception %s' % sys.exc_info()[0])
app.logger.exception(msg)
return flask.jsonify(utilities.make_error(msg))
def get_version():
"""Computes the response of the '/version' endpoint.
Returns:
The value of the docker.get_version() or an error message.
"""
gs = app.context_graph_global_state
try:
version = docker.get_version(gs)
return flask.jsonify(utilities.make_response(version, 'version'))
except collector_error.CollectorError as e:
return flask.jsonify(utilities.make_error(str(e)))
except:
msg = ('get_version() failed with exception %s' % sys.exc_info()[0])
app.logger.exception(msg)
return flask.jsonify(utilities.make_error(msg))
def update_task(task_def):
version = get_version(env)
yes = env.get("yes") or False
image = image_name_from_version(version, env.DOCKER_REPO)
task_def = mutate_image(task_def, image)
print("Updated the definition to the following:\n%s\n" % pformat(task_def))
if not yes: confirm("Continue?")
task_json = json.dumps(task_def, separators=(',', ':'))
created = check_output([
"aws", "ecs", "register-task-definition", "--cli-input-json", task_json
])
rev = json.loads(created)["taskDefinition"]["revision"]
env.revision = rev
print "Success! Created revision %d" % rev
def update_task_def(task_def=None):
"""Copies the most recent task definition (in ECS) to use a given docker image version. Reqs: version"""
if task_def:
execute(update_task, task_def=task_def)
return
version = get_version(env)
task_defs_cmd = capture(
"aws ecs list-task-definitions --family-prefix %s --sort DESC" %
env.APP)
all_defs = json.loads(task_defs_cmd)["taskDefinitionArns"]
app_defs = filter(lambda d: ("task-definition/%s" % env.APP) in d,
all_defs)
last_def = sorted(app_defs, key=task_def_revision)[-1]
last_pretty = pretty_def(last_def)
yes = env.get("yes") or False
print(
"Creating definition based on\n\n\t%(last_pretty)s\n\nusing docker image version %(version)s"
% locals())
if not yes: confirm("Continue?")
old_def = json.loads(
capture("aws ecs describe-task-definition --task-definition %s" %
last_def))
new_def = updated_def_from_old(
old_def, image_name_from_version(version, env.DOCKER_REPO))
print("Updated the definition to the following:\n%s\n" % pformat(new_def))
if not yes: confirm("Continue?")
new_def_json = json.dumps(new_def, separators=(',', ':'))
created = check_output([
"aws", "ecs", "register-task-definition", "--cli-input-json",
new_def_json
])
rev = json.loads(created)["taskDefinition"]["revision"]
env.revision = rev
print "Success! Created revision %d" % rev
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_version(docker.get_version(gs))
g.set_metadata({'timestamp': datetime.datetime.now().isoformat()})
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
nodes_list[0]['timestamp'], {})
# Nodes
for node in nodes_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_node,
{'gs': gs, 'input_queue': input_queue, 'cluster_guid': cluster_guid,
'node': node, 'g': g}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((
gs.get_random_priority(),
_do_compute_service,
{'gs': gs, 'cluster_guid': cluster_guid, 'service': service, 'g': g}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_rcontroller,
{'gs': gs, 'cluster_guid': cluster_guid, 'rcontroller': rcontroller,
'g': g}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Dump the resulting graph
return g.dump(gs, output_format)
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_version(docker.get_version(gs))
g.set_metadata({'timestamp': utilities.now()})
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
input_queue.put((gs.get_random_priority(), _do_compute_node, {
'gs': gs,
'input_queue': input_queue,
'cluster_guid': cluster_guid,
'node': node,
'g': g
}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((gs.get_random_priority(), _do_compute_service, {
'gs': gs,
'cluster_guid': cluster_guid,
'service': service,
'g': g
}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((gs.get_random_priority(), _do_compute_rcontroller, {
'gs': gs,
'cluster_guid': cluster_guid,
'rcontroller': rcontroller,
'g': g
}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
# Dump the resulting graph
return g.dump(gs, output_format)
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
try:
version = docker.get_version(gs)
except Exception as e:
exc_type, value, _ = sys.exc_info()
msg = ('get_version() failed with exception %s: %s' %
(exc_type, value))
gs.logger_error(msg)
version = '_unknown_'
g.set_version(version)
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_node,
{'gs': gs, 'input_queue': input_queue, 'cluster_guid': cluster_guid,
'node': node, 'g': g}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((
gs.get_random_priority(),
_do_compute_service,
{'gs': gs, 'cluster_guid': cluster_guid, 'service': service, 'g': g}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((
gs.get_random_priority(),
_do_compute_rcontroller,
{'gs': gs, 'cluster_guid': cluster_guid, 'rcontroller': rcontroller,
'g': g}))
# Pods running on the master node.
input_queue.put((
gs.get_random_priority(),
_do_compute_master_pods,
{'gs': gs, 'cluster_guid': cluster_guid, 'nodes_list': nodes_list,
'oldest_timestamp': oldest_timestamp, 'g': g}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
g.set_metadata({'timestamp': g.max_resources_and_relations_timestamp()})
# Dump the resulting graph
return g.dump(gs, output_format)