def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
proxy_old_value = None
if os.environ.get("http_proxy"):
proxy_old_value = os.environ["http_proxy"]
logger.warning("http_proxy variable is already set with "
"value: {0}. Change to {1}. Old value "
"will be restored after exit from script's "
"execution context"
.format(proxy_old_value, proxy))
os.environ["http_proxy"] = proxy
try:
yield
except Exception as e:
logger.exception("Error while talking to proxy. Details: {0}"
.format(six.text_type(e)))
finally:
if proxy_old_value:
logger.info("Restoring old value for http_proxy")
os.environ["http_proxy"] = proxy_old_value
else:
logger.info("Deleting set http_proxy environment variable")
del os.environ["http_proxy"]
def handle_task(self, cluster, **kwargs):
if objects.Release.is_lcm_supported(cluster.release):
# this code is actual only if cluster is LCM ready
try:
transaction_options = self.get_transaction_options(
cluster, kwargs
)
except errors.NailgunException as e:
logger.exception("Failed to get transaction options.")
raise self.http(400, six.text_type(e))
if transaction_options:
return self.start_transaction(cluster, transaction_options)
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes, **kwargs)
except Exception as exc:
logger.exception(
u'Cannot execute %s task nodes: %s',
self.task_manager.__name__, ','.join(n.uid for n in nodes)
)
raise self.http(400, msg=six.text_type(exc))
self.raise_task(task)
def _execute_async(self, supertask_id, deployment_tasks=None,
nodes_to_provision_deploy=None):
"""Function for execute task in the mule
:param supertask_id: id of parent task
"""
logger.info(u"ApplyChangesTask: execute async starting for task %s",
supertask_id)
supertask = objects.Task.get_by_uid(supertask_id)
try:
self._execute_async_content(
supertask,
deployment_tasks=deployment_tasks,
nodes_to_provision_deploy=nodes_to_provision_deploy)
except Exception as e:
logger.exception('Error occurred when running task')
data = {
'status': consts.TASK_STATUSES.error,
'progress': 100,
'message': u'Error occurred when running task: {0}'.format(
e.message),
}
objects.Task.update(supertask, data)
db().commit()
def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
proxy_old_value = None
if os.environ.get("http_proxy"):
proxy_old_value = os.environ["http_proxy"]
logger.warning("http_proxy variable is already set with "
"value: {0}. Change to {1}. Old value "
"will be restored after exit from script's "
"execution context".format(proxy_old_value, proxy))
os.environ["http_proxy"] = proxy
try:
yield
except Exception as e:
logger.exception("Error while talking to proxy. Details: {0}".format(
six.text_type(e)))
finally:
if proxy_old_value:
logger.info("Restoring old value for http_proxy")
os.environ["http_proxy"] = proxy_old_value
else:
logger.info("Deleting set http_proxy environment variable")
del os.environ["http_proxy"]
def handle_errors(func, cls, *args, **kwargs):
try:
return func(cls, *args, **kwargs)
except web.HTTPError as http_error:
if http_error.status_code != 204:
web.header('Content-Type', 'application/json', unique=True)
if http_error.status_code >= 400:
http_error.data = json_resp({
"message": http_error.data,
"errors": http_error.err_list
})
else:
http_error.data = json_resp(http_error.data)
raise
except errors.NailgunException as exc:
logger.exception('NailgunException occured')
http_error = BaseHandler.http(400, exc.message)
web.header('Content-Type', 'text/plain')
raise http_error
# intercepting all errors to avoid huge HTML output
except Exception as exc:
logger.exception('Unexpected exception occured')
http_error = BaseHandler.http(
500,
(
traceback.format_exc(exc)
if settings.DEVELOPMENT
else 'Unexpected exception, please check logs'
)
)
http_error.data = json_resp(http_error.data)
web.header('Content-Type', 'text/plain')
raise http_error
def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
variable_values = {
'http_proxy': os.environ.get('http_proxy'),
'https_proxy': os.environ.get('https_proxy')
}
for variable_name, variable_value in variable_values.items():
if os.environ.get(variable_name):
logger.warning("{0} variable is already set with "
"value: {1}. Changing to {2}. Old value "
"will be restored after exit from script's "
"execution context"
.format(variable_name, variable_value, proxy))
os.environ[variable_name] = proxy
try:
yield
except Exception as e:
logger.exception("Error while talking to proxy. Details: {0}"
.format(six.text_type(e)))
finally:
for variable_name, variable_value in variable_values.items():
if variable_value:
logger.info("Restoring old value for http_proxy")
os.environ[variable_name] = variable_value
else:
logger.info("Deleting set {0} environment variable"
.format(variable_name))
del os.environ[variable_name]
def collect(resource_type):
try:
operational_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.operational).all()
error_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.error).all()
all_envs_last_recs = \
OpenStackWorkloadStatsCollection.get_last_by_resource_type(
resource_type)
ready_or_error_ids = set([c.id for c in operational_clusters] +
[c.id for c in error_clusters])
envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \
ready_or_error_ids
# Clear current resource data for unavailable clusters.
# Current OSWL data is cleared for those clusters which status is not
# 'operational' nor 'error' or when cluster was removed. Data is
# cleared for cluster only if it was updated recently (today or
# yesterday). While this collector is running with interval much
# smaller than one day it should not miss any unavailable cluster.
for id in envs_ids_to_clear:
oswl_statistics_save(id, resource_type, [])
# Collect current OSWL data and update data in DB
for cluster in operational_clusters:
try:
client_provider = helpers.ClientProvider(cluster)
proxy_for_os_api = utils.get_proxy_for_cluster(cluster)
version_info = utils.get_version_info(cluster)
with utils.set_proxy(proxy_for_os_api):
data = helpers.get_info_from_os_resource_manager(
client_provider, resource_type)
oswl_statistics_save(cluster.id, resource_type, data,
version_info=version_info)
except errors.StatsException as e:
logger.error("Cannot collect OSWL resource {0} for cluster "
"with id {1}. Details: {2}."
.format(resource_type,
cluster.id,
six.text_type(e))
)
except Exception as e:
logger.exception("Error while collecting OSWL resource {0} "
"for cluster with id {1}. Details: {2}."
.format(resource_type,
cluster.id,
six.text_type(e))
)
db.commit()
except Exception as e:
logger.exception("Exception while collecting OS workloads "
"for resource name {0}. Details: {1}"
.format(resource_type, six.text_type(e)))
finally:
db.remove()
def make_ubuntu_preferences_task(uids, repo):
# NOTE(ikalnitsky): In order to implement the proper pinning,
# we have to download and parse the repo's "Release" file.
# Generally, that's not a good idea to make some HTTP request
# from Nailgun, but taking into account that this task
# will be executed in uWSGI's mule worker we can skip this
# rule, because proper pinning is more valuable thing right now.
template = '\n'.join([
'Package: *',
'Pin: release {conditions}',
'Pin-Priority: {priority}'])
preferences_content = []
try:
release = debian.get_release_file(repo, retries=3)
release = debian.parse_release_file(release)
pin = debian.get_apt_preferences_line(release)
except requests.exceptions.HTTPError as exc:
logger.error("Failed to fetch 'Release' file due to '%s'. "
"The apt preferences won't be applied for repo '%s'.",
six.text_type(exc), repo['name'])
return None
except Exception:
logger.exception("Failed to parse 'Release' file.")
return None
# NOTE(kozhukalov): When a package is available both in:
# 1) http://archive.ubuntu.com/ubuntu trusty universe
# 2) http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0 mos7.0 main
# And if the content of the preferences file is (i.e. by section priority):
# Package: *
# Pin: release o=Mirantis, a=mos7.0, n=mos7.0, l=mos7.0, c=main
# Pin-Priority: 1050
# then the package available in MOS won't match the pin because for
# some reason apt still thinks this package is in universe section.
# As a result:
# # apt-cache policy ohai
# ohai:
# Installed: (none)
# Candidate: 6.14.0-2
# Version table:
# 6.14.0-2 0
# 500 http://10.20.0.1/mirror/ubuntu/ trusty/universe amd64 Packages
# 6.14.0-2~u14.04+mos1 0
# 500 http://10.20.0.2:8080/2015.1.0-7.0/ubuntu/x86_64/ mos7.0/main
# amd64 Packages
preferences_content.append(template.format(
conditions=pin,
priority=repo['priority']))
preferences_content = '\n\n'.join(preferences_content)
preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name'])
return make_upload_task(uids, preferences_content, preferences_path)
def content_json(func, cls, *args, **kwargs):
json_resp = lambda data: (jsonutils.dumps(data) if isinstance(
data, (dict, list)) or data is None else data)
request_validate_needed = True
response_validate_needed = True
resource_type = "single"
if issubclass(cls.__class__,
CollectionHandler) and not func.func_name == "POST":
resource_type = "collection"
if (func.func_name in ("GET", "DELETE")
or getattr(cls.__class__, 'validator', None) is None
or resource_type == "single" and not cls.validator.single_schema
or resource_type == "collection"
and not cls.validator.collection_schema):
request_validate_needed = False
try:
if request_validate_needed:
BaseHandler.checked_data(cls.validator.validate_request,
resource_type=resource_type)
resp = func(cls, *args, **kwargs)
except web.notmodified:
raise
except web.HTTPError as http_error:
if http_error.status_code != 204:
web.header('Content-Type', 'application/json', unique=True)
if http_error.status_code >= 400:
http_error.data = json_resp({
"message": http_error.data,
"errors": http_error.err_list
})
else:
http_error.data = json_resp(http_error.data)
raise
# intercepting all errors to avoid huge HTML output
except Exception as exc:
logger.exception('Unexpected exception occured')
http_error = BaseHandler.http(
500, (traceback.format_exc(exc) if settings.DEVELOPMENT else
'Unexpected exception, please check logs'))
http_error.data = json_resp(http_error.data)
web.header('Content-Type', 'text/plain')
raise http_error
if all([
settings.DEVELOPMENT, response_validate_needed,
getattr(cls.__class__, 'validator', None) is not None
]):
BaseHandler.checked_data(cls.validator.validate_response,
resource_type=resource_type)
web.header('Content-Type', 'application/json', unique=True)
return json_resp(resp)
def make_ubuntu_preferences_task(uids, repo):
# NOTE(ikalnitsky): In order to implement the proper pinning,
# we have to download and parse the repo's "Release" file.
# Generally, that's not a good idea to make some HTTP request
# from Nailgun, but taking into account that this task
# will be executed in uWSGI's mule worker we can skip this
# rule, because proper pinning is more valuable thing right now.
template = '\n'.join([
'Package: *',
'Pin: release {conditions}',
'Pin-Priority: {priority}'])
preferences_content = []
try:
release = debian.get_release_file(repo, retries=3)
release = debian.parse_release_file(release)
pin = debian.get_apt_preferences_line(release)
except requests.exceptions.HTTPError as exc:
logger.error("Failed to fetch 'Release' file due to '%s'. "
"The apt preferences won't be applied for repo '%s'.",
six.text_type(exc), repo['name'])
return None
except Exception:
logger.exception("Failed to parse 'Release' file.")
return None
# NOTE(kozhukalov): When a package is available both in:
# 1) http://archive.ubuntu.com/ubuntu trusty universe
# 2) http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0 mos7.0 main
# And if the content of the preferences file is (i.e. by section priority):
# Package: *
# Pin: release o=Mirantis, a=mos7.0, n=mos7.0, l=mos7.0, c=main
# Pin-Priority: 1050
# then the package available in MOS won't match the pin because for
# some reason apt still thinks this package is in universe section.
# As a result:
# # apt-cache policy ohai
# ohai:
# Installed: (none)
# Candidate: 6.14.0-2
# Version table:
# 6.14.0-2 0
# 500 http://10.20.0.1/mirror/ubuntu/ trusty/universe amd64 Packages
# 6.14.0-2~u14.04+mos1 0
# 500 http://10.20.0.2:8080/2015.1.0-7.0/ubuntu/x86_64/ mos7.0/main
# amd64 Packages
preferences_content.append(template.format(
conditions=pin,
priority=repo['priority']))
preferences_content = '\n\n'.join(preferences_content)
preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name'])
return make_upload_task(uids, preferences_content, preferences_path)
def PUT(self, cluster_id):
""":returns: JSONized Task object.
:http: * 202 (task successfully executed)
* 400 (invalid object data specified)
* 404 (environment is not found)
* 409 (task with such parameters already exists)
"""
cluster = self.get_object_or_404(
objects.Cluster,
cluster_id,
log_404=(u"warning", u"Error: there is no cluster "
u"with id '{0}' in DB.".format(cluster_id)))
logger.info(self.log_message.format(env_id=cluster_id))
try:
options = self.get_options()
except ValueError as e:
raise self.http(400, six.text_type(e))
try:
self.validator.validate(cluster)
except errors.NailgunException as e:
raise self.http(400, e.message)
if objects.Release.is_lcm_supported(cluster.release):
# try to get new graph to run transaction manager
try:
transaction_options = self.get_transaction_options(
cluster, options)
except errors.NailgunException as e:
logger.exception("Failed to get transaction options")
raise self.http(400, msg=six.text_type(e))
if transaction_options:
return self.start_transaction(cluster, transaction_options)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(**options)
except (errors.AlreadyExists, errors.StopAlreadyRunning) as exc:
raise self.http(409, exc.message)
except (
errors.DeploymentNotRunning,
errors.NoDeploymentTasks,
errors.WrongNodeStatus,
errors.UnavailableRelease,
errors.CannotBeStopped,
) as exc:
raise self.http(400, exc.message)
except Exception as exc:
logger.error(
self.log_error.format(env_id=cluster_id, error=str(exc)))
# let it be 500
raise
self.raise_task(task)
def collect(resource_type):
try:
operational_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.operational).all()
error_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.error).all()
all_envs_last_recs = \
OpenStackWorkloadStatsCollection.get_last_by_resource_type(
resource_type)
ready_or_error_ids = set([c.id for c in operational_clusters] +
[c.id for c in error_clusters])
envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \
ready_or_error_ids
# Clear current resource data for unavailable clusters.
# Current OSWL data is cleared for those clusters which status is not
# 'operational' nor 'error' or when cluster was removed. Data is
# cleared for cluster only if it was updated recently (today or
# yesterday). While this collector is running with interval much
# smaller than one day it should not miss any unavailable cluster.
for id in envs_ids_to_clear:
oswl_statistics_save(id, resource_type, [])
# Collect current OSWL data and update data in DB
for cluster in operational_clusters:
try:
client_provider = helpers.ClientProvider(cluster)
proxy_for_os_api = utils.get_proxy_for_cluster(cluster)
version_info = utils.get_version_info(cluster)
with utils.set_proxy(proxy_for_os_api):
data = helpers.get_info_from_os_resource_manager(
client_provider, resource_type)
oswl_statistics_save(cluster.id,
resource_type,
data,
version_info=version_info)
except errors.StatsException as e:
logger.error("Cannot collect OSWL resource {0} for cluster "
"with id {1}. Details: {2}.".format(
resource_type, cluster.id, six.text_type(e)))
except Exception as e:
logger.exception(
"Error while collecting OSWL resource {0} "
"for cluster with id {1}. Details: {2}.".format(
resource_type, cluster.id, six.text_type(e)))
db.commit()
except Exception as e:
logger.exception("Exception while collecting OS workloads "
"for resource name {0}. Details: {1}".format(
resource_type, six.text_type(e)))
finally:
db.remove()
def _distributed_serialize_tasks_for_node(formatter_contexts_idx,
node_and_tasks, scattered_data):
"""Remote serialization call for DistributedProcessingPolicy
Code of the function is copied to the workers and executed there, thus
we are including all required imports inside the function.
:param formatter_contexts_idx: dict of formatter contexts with node_id
value as key
:param node_and_tasks: list of node_id, task_data tuples
:param scattered_data: feature object, that points to data copied to
workers
:return: [(node_id, serialized), error]
"""
try:
factory = TasksSerializersFactory(scattered_data['context'])
# Restoring settings
settings.config = scattered_data['settings_config']
for k in formatter_contexts_idx:
formatter_contexts_idx[k]['SETTINGS'] = settings
except Exception as e:
logger.exception("Distributed serialization failed")
return [((None, None), e)]
result = []
for node_and_task in node_and_tasks:
node_id = None
try:
node_id, task = node_and_task
logger.debug("Starting distributed node %s task %s serialization",
node_id, task['id'])
formatter_context = formatter_contexts_idx[node_id]
serializer = factory.create_serializer(task)
serialized = serializer.serialize(
node_id, formatter_context=formatter_context)
logger.debug(
"Distributed node %s task %s serialization "
"result: %s", node_id, task['id'], serialized)
result.append(((node_id, serialized), None))
except Exception as e:
logger.exception("Distributed serialization failed")
result.append(((node_id, None), e))
break
logger.debug("Processed tasks count: %s", len(result))
return result
def _distributed_serialize_tasks_for_node(formatter_contexts_idx,
node_and_tasks, scattered_data):
"""Remote serialization call for DistributedProcessingPolicy
Code of the function is copied to the workers and executed there, thus
we are including all required imports inside the function.
:param formatter_contexts_idx: dict of formatter contexts with node_id
value as key
:param node_and_tasks: list of node_id, task_data tuples
:param scattered_data: feature object, that points to data copied to
workers
:return: [(node_id, serialized), error]
"""
try:
factory = TasksSerializersFactory(scattered_data['context'])
# Restoring settings
settings.config = scattered_data['settings_config']
for k in formatter_contexts_idx:
formatter_contexts_idx[k]['SETTINGS'] = settings
except Exception as e:
logger.exception("Distributed serialization failed")
return [((None, None), e)]
result = []
for node_and_task in node_and_tasks:
node_id = None
try:
node_id, task = node_and_task
logger.debug("Starting distributed node %s task %s serialization",
node_id, task['id'])
formatter_context = formatter_contexts_idx[node_id]
serializer = factory.create_serializer(task)
serialized = serializer.serialize(
node_id, formatter_context=formatter_context)
logger.debug("Distributed node %s task %s serialization "
"result: %s", node_id, task['id'], serialized)
result.append(((node_id, serialized), None))
except Exception as e:
logger.exception("Distributed serialization failed")
result.append(((node_id, None), e))
break
logger.debug("Processed tasks count: %s", len(result))
return result
def _raise_error_task(self, cluster, task_name, exc):
# set task status to error and update its corresponding data
task = Task(
name=task_name, cluster=cluster, status=consts.TASK_STATUSES.error, progress=100, message=six.text_type(exc)
)
db().add(task)
db().commit()
logger.exception("Error in network configuration")
self.raise_task(task)
def _serialize_task_for_node(factory, node_and_task):
node_id, task = node_and_task
logger.debug("applying task '%s' for node: %s", task['id'], node_id)
try:
task_serializer = factory.create_serializer(task)
serialized = task_serializer.serialize(node_id)
return node_id, serialized
except Exception:
logger.exception("failed to serialize task '%s' for node: %s",
task['id'], node_id)
raise
def must_send_stats(self):
try:
stat_settings = objects.MasterNodeSettings.get_one(). \
settings.get("statistics", {})
return stat_settings.get("user_choice_saved", {}).\
get("value", False) and \
stat_settings.get("send_anonymous_statistic", {}). \
get("value", False)
except Exception as e:
logger.exception(
"Get statistics settings failed: %s", six.text_type(e))
return False
def must_send_stats(self):
try:
stat_settings = getattr(objects.MasterNodeSettings.get_one(),
"settings", {}).get("statistics", {})
return stat_settings.get("user_choice_saved", {}).\
get("value", False) and \
stat_settings.get("send_anonymous_statistic", {}). \
get("value", False)
except (AttributeError, TypeError) as e:
logger.exception("Get statistics settings failed: %s",
six.text_type(e))
return False
def ping_collector(self):
try:
resp = requests.get(settings.COLLECTOR_PING_URL,
timeout=settings.COLLECTOR_RESP_TIMEOUT)
resp.raise_for_status()
return True
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.RequestException,
requests.exceptions.HTTPError) as e:
logger.exception("Collector ping failed: %s", six.text_type(e))
return False
def _raise_error_task(self, cluster, task_name, exc):
# set task status to error and update its corresponding data
task = Task(name=task_name,
cluster=cluster,
status=consts.TASK_STATUSES.error,
progress=100,
message=six.text_type(exc))
db().add(task)
db().commit()
logger.exception('Error in network configuration')
self.raise_task(task)
def _handle_stats_opt_in(self):
if self.single.must_send_stats():
logger.debug("Handling customer opt-in to sending statistics")
manager = CreateStatsUserTaskManager()
else:
logger.debug("Handling customer opt-out to sending statistics")
manager = RemoveStatsUserTaskManager()
try:
manager.execute()
except Exception:
logger.exception("Stats user operation failed")
def _serialize_task_for_node(factory, node_and_task):
node_id, task = node_and_task
logger.debug(
"applying task '%s' for node: %s", task['id'], node_id
)
try:
task_serializer = factory.create_serializer(task)
serialized = task_serializer.serialize(node_id)
return node_id, serialized
except Exception:
logger.exception(
"failed to serialize task '%s' for node: %s", task['id'], node_id
)
raise
def set_default_node_volumes(cls, node):
from .objects.volumes import VolumeObject
try:
VolumeObject.set_default_node_volumes(node)
except Exception as exc:
logger.exception(exc)
msg = "Failed to generate volumes for node '{0}': '{1}'".format(
node.human_readable_name, six.text_type(exc)
)
Notification.create({"topic": "error", "message": msg, "node_id": node.id})
if node.cluster_id:
Node.add_pending_change(node, "disks")
def ping_collector(self):
try:
resp = requests.get(self.build_collector_url("COLLECTOR_PING_URL"),
timeout=settings.COLLECTOR_RESP_TIMEOUT)
resp.raise_for_status()
return True
except (urllib3.exceptions.DecodeError, urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects,
requests.exceptions.HTTPError) as e:
logger.error("Collector ping failed: %s", type(e).__name__)
except Exception as e:
logger.exception("Collector ping failed: %s", six.text_type(e))
return False
def remove_silently(path):
"""Removes an element from file system
no matter if it's file, folder or symlink. Ignores OSErrors.
:param path: path
"""
try:
if os.path.islink(path):
os.unlink(path)
elif os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except OSError as e:
logger.exception(e)
def set_default_node_volumes(cls, node):
from .objects.volumes import VolumeObject
try:
VolumeObject.set_default_node_volumes(node)
except Exception as exc:
logger.exception(exc)
msg = "Failed to generate volumes for node '{0}': '{1}'".format(
node.human_readable_name, six.text_type(exc))
objects.Notification.create({
'topic': 'error',
'message': msg,
'node_id': node.id})
if node.cluster_id:
objects.Node.add_pending_change(node, 'disks')
def set_proxy(self):
if os.environ.get("http_proxy"):
raise Exception("Cannot set 'http_proxy' environment variable "
"as it already has a value")
os.environ["http_proxy"] = self.proxy
try:
yield
except Exception as e:
logger.exception("Error while interacting with "
"OpenStack api. Details: {0}".format(
six.text_type(e)))
finally:
if os.environ.get("http_proxy") == self.proxy:
del (os.environ["http_proxy"])
def remove_silently(path):
"""Removes an element from file system
no matter if it's file, folder or symlink. Ignores OSErrors.
:param path: path
"""
try:
if os.path.islink(path):
os.unlink(path)
elif os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except OSError as e:
logger.exception(e)
def ping_collector(self):
try:
resp = requests.get(self.build_collector_url("COLLECTOR_PING_URL"),
timeout=settings.COLLECTOR_RESP_TIMEOUT)
resp.raise_for_status()
return True
except (urllib3.exceptions.DecodeError,
urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects,
requests.exceptions.HTTPError) as e:
logger.error("Collector ping failed: %s", type(e).__name__)
except Exception as e:
logger.exception("Collector ping failed: %s", six.text_type(e))
return False
def delete_expired_oswl_entries():
try:
deleted_rows_count = \
objects.OpenStackWorkloadStatsCollection.clean_expired_entries()
if deleted_rows_count == 0:
logger.info("There are no expired OSWL entries in db.")
db().commit()
logger.info("Expired OSWL entries are " "successfully cleaned from db")
except Exception as e:
logger.exception("Exception while cleaning oswls entries from "
"db. Details: {0}".format(six.text_type(e)))
finally:
db.remove()
def prepare_action_log_kwargs(cls, task):
"""Prepares kwargs dict for ActionLog db model class
:param task: task instance to be processed
:returns: kwargs dict for action log creation
"""
create_kwargs = {
'task_uuid': task.uuid,
'cluster_id': task.cluster_id,
'action_group': tasks_names_actions_groups_mapping[task.name],
'action_name': task.name,
'action_type': consts.ACTION_TYPES.nailgun_task,
'start_timestamp': datetime.datetime.utcnow()
}
# actor_id passed from ConnectionMonitor middleware and is
# needed for binding task execution event with particular
# actor
actor_id = None
try:
actor_id_field = 'fuel.action.actor_id'
if hasattr(web.ctx, 'env') and actor_id_field in web.ctx.env:
# Fetching actor_id from env context
actor_id = web.ctx.env.get(actor_id_field)
else:
# Fetching actor_id from parent task action log
from nailgun import objects # preventing cycle import error
if task.parent_id:
parent_task = objects.Task.get_by_uid(task.parent_id)
action_log = objects.ActionLog.get_by_kwargs(
task_uuid=parent_task.uuid,
action_name=parent_task.name)
actor_id = action_log.actor_id
except Exception:
logger.exception("Extracting of actor_id failed")
create_kwargs['actor_id'] = actor_id
additional_info = {
'parent_task_id': task.parent_id,
'subtasks_ids': [t.id for t in task.subtasks],
'operation': task.name
}
create_kwargs['additional_info'] = additional_info
return create_kwargs
def prepare_action_log_kwargs(cls, task):
"""Prepares kwargs dict for ActionLog db model class
:param task: task instance to be processed
:returns: kwargs dict for action log creation
"""
create_kwargs = {
'task_uuid': task.uuid,
'cluster_id': task.cluster_id,
'action_group': tasks_names_actions_groups_mapping[task.name],
'action_name': task.name,
'action_type': consts.ACTION_TYPES.nailgun_task,
'start_timestamp': datetime.datetime.utcnow()
}
# actor_id passed from ConnectionMonitor middleware and is
# needed for binding task execution event with particular
# actor
actor_id = None
try:
actor_id_field = 'fuel.action.actor_id'
if hasattr(web.ctx, 'env') and actor_id_field in web.ctx.env:
# Fetching actor_id from env context
actor_id = web.ctx.env.get(actor_id_field)
else:
# Fetching actor_id from parent task action log
from nailgun import objects # preventing cycle import error
if task.parent_id:
parent_task = objects.Task.get_by_uid(task.parent_id)
action_log = objects.ActionLog.get_by_kwargs(
task_uuid=parent_task.uuid,
action_name=parent_task.name)
actor_id = action_log.actor_id
except Exception:
logger.exception("Extracting of actor_id failed")
create_kwargs['actor_id'] = actor_id
additional_info = {
'parent_task_id': task.parent_id,
'subtasks_ids': [t.id for t in task.subtasks],
'operation': task.name
}
create_kwargs['additional_info'] = additional_info
return create_kwargs
def send_serialized(records):
if records:
logger.info("Send %d records", len(records))
try:
req_body = {"action_logs": records}
headers = {'content-type': 'application/json'}
resp = requests.post(
settings.COLLECTOR_STORE_URL,
headers=headers,
data=jsonutils.dumps(req_body),
timeout=settings.COLLECTOR_RESP_TIMEOUT)
resp.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.RequestException,
requests.exceptions.HTTPError) as e:
logger.exception(
"Action logs sending to collector failed: %s",
six.text_type(e))
else:
resp_dict = resp.json()
if resp.status_code == requests.codes.created and \
resp_dict["status"] == \
consts.LOG_CHUNK_SEND_STATUS.ok:
records_resp = resp_dict["action_logs"]
saved_ids = set()
failed_ids = set()
for record in records_resp:
if record["status"] == \
consts.LOG_RECORD_SEND_STATUS.failed:
failed_ids.add(record["external_id"])
else:
saved_ids.add(record["external_id"])
sent_saved_ids = set(saved_ids) & set(ids)
logger.info("Records saved: %s, failed: %s",
six.text_type(list(sent_saved_ids)),
six.text_type(list(failed_ids)))
db().query(models.ActionLog).filter(
models.ActionLog.id.in_(sent_saved_ids)
).update(
{"is_sent": True}, synchronize_session=False
)
db().commit()
else:
logger.error("Unexpected collector answer: %s",
six.text_type(resp))
def send_data_to_url(self, url, data):
resp = None
try:
headers = {"content-type": "application/json", "master-node-uid": InstallationInfo().get_master_node_uid()}
resp = requests.post(
url, headers=headers, data=jsonutils.dumps(data), timeout=settings.COLLECTOR_RESP_TIMEOUT
)
except (
urllib3.exceptions.DecodeError,
urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects,
) as e:
logger.error("Sending data to collector failed: %s", type(e).__name__)
except Exception as e:
logger.exception("Sending data to collector failed: %s", six.text_type(e))
return resp
def send_data_to_url(self, url, data):
resp = None
try:
headers = {'content-type': 'application/json'}
resp = requests.post(url,
headers=headers,
data=jsonutils.dumps(data),
timeout=settings.COLLECTOR_RESP_TIMEOUT)
except (urllib3.exceptions.DecodeError, urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects) as e:
logger.error("Sending data to collector failed: %s",
type(e).__name__)
except Exception as e:
logger.exception("Sending data to collector failed: %s",
six.text_type(e))
return resp
def GET(self, cluster_id):
""":returns: JSONized network configuration for cluster.
:http: * 200 (OK)
* 404 (cluster not found in db)
"""
cluster = self.get_object_or_404(objects.Cluster, cluster_id)
self.check_net_provider(cluster)
try:
# there are a plenty of reasons why serializer could throw
# an exception. usually that means we don't handle properly
# some corner cases, and it should be fixed. in order
# to simplify troubleshootng, let's print traceback to log.
return self.serializer.serialize_for_cluster(cluster)
except Exception:
logger.exception('Serialization failed')
raise
def set_proxy(self):
if os.environ.get("http_proxy"):
raise Exception(
"Cannot set 'http_proxy' environment variable "
"as it already has a value"
)
os.environ["http_proxy"] = self.proxy
try:
yield
except Exception as e:
logger.exception("Error while interacting with "
"OpenStack api. Details: {0}"
.format(six.text_type(e)))
finally:
if os.environ.get("http_proxy") == self.proxy:
del(os.environ["http_proxy"])
def get_openstack_info(self, cluster, cluster_nodes):
if not cluster.status == consts.CLUSTER_STATUSES.operational:
return {}
info = {}
try:
getter = \
openstack_info_collector.OpenStackInfoCollector(
cluster,
cluster_nodes
)
info = getter.get_info()
except Exception as e:
logger.exception("Cannot collect OpenStack info due to error: %s",
six.text_type(e))
return info
def get_version_info(cluster):
"""Returns current Fuel and OpenStack version info
:param cluster: cluster
:type cluster: nailgun.db.sqlalchemy.models.Cluster
:return: dict with version info or None
"""
try:
return {
'fuel_version': cluster.fuel_version,
'release_version': cluster.release.version,
'release_name': cluster.release.name,
'release_os': cluster.release.operating_system,
'environment_version': cluster.release.environment_version
}
except Exception:
logger.exception("Fetching version info for cluster '%s' failed",
cluster)
def _handle_stats_opt_in(self, settings_data=None):
"""Starts task on stats user creation or removal
:param settings_data: dict with master node settings.
Current data from DB will be used if master_node_settings_data is None
"""
must_send = self.single.must_send_stats(
master_node_settings_data=settings_data)
if must_send:
logger.debug("Handling customer opt-in to sending statistics")
manager = CreateStatsUserTaskManager()
else:
logger.debug("Handling customer opt-out to sending statistics")
manager = RemoveStatsUserTaskManager()
try:
manager.execute()
except Exception:
logger.exception("Stats user operation failed")
def delete_expired_oswl_entries():
try:
deleted_rows_count = \
objects.OpenStackWorkloadStatsCollection.clean_expired_entries()
if deleted_rows_count == 0:
logger.info("There are no expired OSWL entries in db.")
db().commit()
logger.info("Expired OSWL entries are "
"successfully cleaned from db")
except Exception as e:
logger.exception("Exception while cleaning oswls entries from "
"db. Details: {0}".format(six.text_type(e)))
finally:
db.remove()
def _execute_async(self, supertask_id):
logger.info(u"ApplyChangesTask: execute async starting for task %s",
supertask_id)
supertask = objects.Task.get_by_uid(supertask_id)
try:
self._execute_async_content(supertask)
except Exception as e:
logger.exception('Error occurred when running task')
data = {
'status': consts.TASK_STATUSES.error,
'progress': 100,
'message': u'Error occurred when running task: {0}'.format(
e.message),
}
objects.Task.update(supertask, data)
db().commit()
def GET(self, cluster_id):
""":returns: JSONized network configuration for cluster.
:http: * 200 (OK)
* 404 (cluster not found in db)
"""
cluster = self.get_object_or_404(objects.Cluster, cluster_id)
self.check_net_provider(cluster)
try:
# there are a plenty of reasons why serializer could throw
# an exception. usually that means we don't handle properly
# some corner cases, and it should be fixed. in order
# to simplify troubleshootng, let's print traceback to log.
return self.serializer.serialize_for_cluster(cluster)
except Exception:
logger.exception('Serialization failed')
raise
def make_ubuntu_preferences_task(uids, repo):
# NOTE(ikalnitsky): In order to implement the proper pinning,
# we have to download and parse the repo's "Release" file.
# Generally, that's not a good idea to make some HTTP request
# from Nailgun, but taking into account that this task
# will be executed in uWSGI's mule worker we can skip this
# rule, because proper pinning is more valuable thing right now.
template = '\n'.join([
'Package: *',
'Pin: release {conditions}',
'Pin-Priority: {priority}'])
preferences_content = []
try:
release = debian.get_release_file(repo, retries=3)
release = debian.parse_release_file(release)
pin = debian.get_apt_preferences_line(release)
except requests.exceptions.HTTPError as exc:
logger.error("Failed to fetch 'Release' file due to '%s'. "
"The apt preferences won't be applied for repo '%s'.",
six.text_type(exc), repo['name'])
return None
except Exception:
logger.exception("Failed to parse 'Release' file.")
return None
# if sections are detected (non-flat repo) create pinning per
# sections; otherwise - create just one pin for the entire repo
if repo['section']:
for section in repo['section'].split():
preferences_content.append(template.format(
conditions='{0},c={1}'.format(pin, section),
priority=repo['priority']))
else:
preferences_content.append(template.format(
conditions=pin,
priority=repo['priority']))
preferences_content = '\n\n'.join(preferences_content)
preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name'])
return make_upload_task(uids, preferences_content, preferences_path)
def make_ubuntu_preferences_task(uids, repo):
# NOTE(ikalnitsky): In order to implement the proper pinning,
# we have to download and parse the repo's "Release" file.
# Generally, that's not a good idea to make some HTTP request
# from Nailgun, but taking into account that this task
# will be executed in uWSGI's mule worker we can skip this
# rule, because proper pinning is more valuable thing right now.
template = '\n'.join([
'Package: *',
'Pin: release {conditions}',
'Pin-Priority: {priority}'])
preferences_content = []
try:
release = debian.get_release_file(repo, retries=3)
release = debian.parse_release_file(release)
pin = debian.get_apt_preferences_line(release)
except requests.exceptions.HTTPError as exc:
logger.error("Failed to fetch 'Release' file due to '%s'. "
"The apt preferences won't be applied for repo '%s'.",
six.text_type(exc), repo['name'])
return None
except Exception:
logger.exception("Failed to parse 'Release' file.")
return None
# if sections are detected (non-flat repo) create pinning per
# sections; otherwise - create just one pin for the entire repo
if repo['section']:
for section in repo['section'].split():
preferences_content.append(template.format(
conditions='{0},c={1}'.format(pin, section),
priority=repo['priority']))
else:
preferences_content.append(template.format(
conditions=pin,
priority=repo['priority']))
preferences_content = '\n\n'.join(preferences_content)
preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name'])
return make_upload_task(uids, preferences_content, preferences_path)
def get_openstack_info(self, cluster, cluster_nodes):
if not cluster.status == consts.CLUSTER_STATUSES.operational:
return {}
info = {}
try:
getter = \
openstack_info_collector.OpenStackInfoCollector(
cluster,
cluster_nodes
)
info = getter.get_info()
except Exception as e:
logger.exception(
"Cannot collect OpenStack info due to error: %s",
six.text_type(e)
)
return info
def collect(resource_type):
try:
operational_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.operational).all()
for cluster in operational_clusters:
client_provider = utils.ClientProvider(cluster)
proxy_for_os_api = utils.get_proxy_for_cluster(cluster)
with utils.set_proxy(proxy_for_os_api):
data = utils.get_info_from_os_resource_manager(
client_provider, resource_type)
oswl_statistics_save(cluster.id, resource_type, data)
db.commit()
except Exception as e:
logger.exception("Exception while collecting OS workloads "
"for resource name {0}. Details: {1}".format(
resource_type, six.text_type(e)))
finally:
db.remove()
def _execute_async(self, supertask_id):
"""Function for execute task in the mule
:param supertask_id: id of parent task
"""
logger.info(u"ApplyChangesTask: execute async starting for task %s",
supertask_id)
supertask = objects.Task.get_by_uid(supertask_id)
try:
self._execute_async_content(supertask)
except Exception as e:
logger.exception('Error occurred when running task')
data = {
'status': consts.TASK_STATUSES.error,
'progress': 100,
'message': u'Error occurred when running task: {0}'.format(
e.message),
}
objects.Task.update(supertask, data)
db().commit()
def _call_silently(self, task, instance, *args, **kwargs):
# create action_log for task
al = TaskHelper.create_action_log(task)
method = getattr(instance, kwargs.pop('method_name', 'execute'))
if task.status == consts.TASK_STATUSES.error:
TaskHelper.update_action_log(task, al)
return
try:
to_return = method(task, *args, **kwargs)
# update action_log instance for task
# for asynchronous task it will be not final update
# as they also are updated in rpc receiver
TaskHelper.update_action_log(task, al)
return to_return
except errors.NoChanges as e:
self._finish_task(task, al, consts.TASK_STATUSES.ready, str(e))
except Exception as exc:
logger.exception("Task '%s' failed.", task.name)
self._finish_task(task, al, consts.TASK_STATUSES.error, str(exc))
def process_task(self, task, node_ids):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param node_ids: the list of nodes, where this tasks should run
"""
logger.debug("applying task '%s' for nodes: %s", task['id'], node_ids)
task_serializer = self.factory.create_serializer(task)
for node_id in node_ids:
try:
task = task_serializer.serialize(node_id)
except Exception:
logger.exception("Failed to serialize task %s", task['id'])
raise
node_tasks = self.tasks_graph.setdefault(node_id, {})
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, task):
node_tasks[task['id']] = task
def _call_silently(self, task, instance, *args, **kwargs):
# create action_log for task
al = TaskHelper.create_action_log(task)
method = getattr(instance, kwargs.pop('method_name', 'execute'))
if task.status == consts.TASK_STATUSES.error:
TaskHelper.update_action_log(task, al)
return
try:
to_return = method(task, *args, **kwargs)
# update action_log instance for task
# for asynchronous task it will be not final update
# as they also are updated in rpc receiver
TaskHelper.update_action_log(task, al)
return to_return
except errors.NoChanges as e:
self._finish_task(task, al, consts.TASK_STATUSES.ready, str(e))
except Exception as exc:
logger.exception("Task '%s' failed.", task.name)
self._finish_task(task, al, consts.TASK_STATUSES.error, str(exc))
def process_task(self, task, node_ids):
"""Processes one task one nodes of cluster.
:param task: the task instance
:param node_ids: the list of nodes, where this tasks should run
"""
logger.debug("applying task '%s' for nodes: %s", task['id'], node_ids)
task_serializer = self.factory.create_serializer(task)
for node_id in node_ids:
try:
task = task_serializer.serialize(node_id)
except Exception:
logger.exception("Failed to serialize task %s", task['id'])
raise
node_tasks = self.tasks_graph.setdefault(node_id, {})
# de-duplication the tasks on node
# since task can be added after expand group need to
# overwrite if existed task is skipped and new is not skipped.
if self.need_update_task(node_tasks, task):
node_tasks[task['id']] = task
def handle_task(self, cluster, **kwargs):
if objects.Release.is_lcm_supported(cluster.release):
# this code is actual only if cluster is LCM ready
try:
transaction_options = self.get_transaction_options(
cluster, kwargs)
except errors.NailgunException as e:
logger.exception("Failed to get transaction options.")
raise self.http(400, six.text_type(e))
if transaction_options:
return self.start_transaction(cluster, transaction_options)
nodes = self.get_nodes(cluster)
try:
task_manager = self.task_manager(cluster_id=cluster.id)
task = task_manager.execute(nodes, **kwargs)
except Exception as exc:
logger.exception(u'Cannot execute %s task nodes: %s',
self.task_manager.__name__,
','.join(n.uid for n in nodes))
raise self.http(400, msg=six.text_type(exc))
self.raise_task(task)
def on_load_failure(manager, endpoint, exc):
logger.exception("Failed to load %s extension", endpoint.name)
raise
def content_json(func, cls, *args, **kwargs):
json_resp = lambda data: (
jsonutils.dumps(data)
if isinstance(data, (dict, list)) or data is None else data
)
request_validate_needed = True
response_validate_needed = True
resource_type = "single"
if issubclass(
cls.__class__,
CollectionHandler
) and not func.func_name == "POST":
resource_type = "collection"
if (
func.func_name in ("GET", "DELETE") or
getattr(cls.__class__, 'validator', None) is None or
resource_type == "single" and not cls.validator.single_schema or
resource_type == "collection" and not cls.validator.collection_schema
):
request_validate_needed = False
try:
if request_validate_needed:
BaseHandler.checked_data(
cls.validator.validate_request,
resource_type=resource_type
)
resp = func(cls, *args, **kwargs)
except web.notmodified:
raise
except web.HTTPError as http_error:
if http_error.status_code != 204:
web.header('Content-Type', 'application/json', unique=True)
if http_error.status_code >= 400:
http_error.data = json_resp({
"message": http_error.data,
"errors": http_error.err_list
})
else:
http_error.data = json_resp(http_error.data)
raise
# intercepting all errors to avoid huge HTML output
except Exception as exc:
logger.exception('Unexpected exception occured')
http_error = BaseHandler.http(
500,
(
traceback.format_exc(exc)
if settings.DEVELOPMENT
else six.text_type(exc)
)
)
http_error.data = json_resp(http_error.data)
raise http_error
if all([
settings.DEVELOPMENT,
response_validate_needed,
getattr(cls.__class__, 'validator', None) is not None
]):
BaseHandler.checked_data(
cls.validator.validate_response,
resource_type=resource_type
)
web.header('Content-Type', 'application/json', unique=True)
return json_resp(resp)
