def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ proxy_old_value = None if os.environ.get("http_proxy"): proxy_old_value = os.environ["http_proxy"] logger.warning("http_proxy variable is already set with " "value: {0}. Change to {1}. Old value " "will be restored after exit from script's " "execution context" .format(proxy_old_value, proxy)) os.environ["http_proxy"] = proxy try: yield except Exception as e: logger.exception("Error while talking to proxy. Details: {0}" .format(six.text_type(e))) finally: if proxy_old_value: logger.info("Restoring old value for http_proxy") os.environ["http_proxy"] = proxy_old_value else: logger.info("Deleting set http_proxy environment variable") del os.environ["http_proxy"]
def handle_task(self, cluster, **kwargs): if objects.Release.is_lcm_supported(cluster.release): # this code is actual only if cluster is LCM ready try: transaction_options = self.get_transaction_options( cluster, kwargs ) except errors.NailgunException as e: logger.exception("Failed to get transaction options.") raise self.http(400, six.text_type(e)) if transaction_options: return self.start_transaction(cluster, transaction_options) nodes = self.get_nodes(cluster) try: task_manager = self.task_manager(cluster_id=cluster.id) task = task_manager.execute(nodes, **kwargs) except Exception as exc: logger.exception( u'Cannot execute %s task nodes: %s', self.task_manager.__name__, ','.join(n.uid for n in nodes) ) raise self.http(400, msg=six.text_type(exc)) self.raise_task(task)
def _execute_async(self, supertask_id, deployment_tasks=None, nodes_to_provision_deploy=None): """Function for execute task in the mule :param supertask_id: id of parent task """ logger.info(u"ApplyChangesTask: execute async starting for task %s", supertask_id) supertask = objects.Task.get_by_uid(supertask_id) try: self._execute_async_content( supertask, deployment_tasks=deployment_tasks, nodes_to_provision_deploy=nodes_to_provision_deploy) except Exception as e: logger.exception('Error occurred when running task') data = { 'status': consts.TASK_STATUSES.error, 'progress': 100, 'message': u'Error occurred when running task: {0}'.format( e.message), } objects.Task.update(supertask, data) db().commit()
def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ proxy_old_value = None if os.environ.get("http_proxy"): proxy_old_value = os.environ["http_proxy"] logger.warning("http_proxy variable is already set with " "value: {0}. Change to {1}. Old value " "will be restored after exit from script's " "execution context".format(proxy_old_value, proxy)) os.environ["http_proxy"] = proxy try: yield except Exception as e: logger.exception("Error while talking to proxy. Details: {0}".format( six.text_type(e))) finally: if proxy_old_value: logger.info("Restoring old value for http_proxy") os.environ["http_proxy"] = proxy_old_value else: logger.info("Deleting set http_proxy environment variable") del os.environ["http_proxy"]
def handle_errors(func, cls, *args, **kwargs): try: return func(cls, *args, **kwargs) except web.HTTPError as http_error: if http_error.status_code != 204: web.header('Content-Type', 'application/json', unique=True) if http_error.status_code >= 400: http_error.data = json_resp({ "message": http_error.data, "errors": http_error.err_list }) else: http_error.data = json_resp(http_error.data) raise except errors.NailgunException as exc: logger.exception('NailgunException occured') http_error = BaseHandler.http(400, exc.message) web.header('Content-Type', 'text/plain') raise http_error # intercepting all errors to avoid huge HTML output except Exception as exc: logger.exception('Unexpected exception occured') http_error = BaseHandler.http( 500, ( traceback.format_exc(exc) if settings.DEVELOPMENT else 'Unexpected exception, please check logs' ) ) http_error.data = json_resp(http_error.data) web.header('Content-Type', 'text/plain') raise http_error
def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ variable_values = { 'http_proxy': os.environ.get('http_proxy'), 'https_proxy': os.environ.get('https_proxy') } for variable_name, variable_value in variable_values.items(): if os.environ.get(variable_name): logger.warning("{0} variable is already set with " "value: {1}. Changing to {2}. Old value " "will be restored after exit from script's " "execution context" .format(variable_name, variable_value, proxy)) os.environ[variable_name] = proxy try: yield except Exception as e: logger.exception("Error while talking to proxy. Details: {0}" .format(six.text_type(e))) finally: for variable_name, variable_value in variable_values.items(): if variable_value: logger.info("Restoring old value for http_proxy") os.environ[variable_name] = variable_value else: logger.info("Deleting set {0} environment variable" .format(variable_name)) del os.environ[variable_name]
def collect(resource_type): try: operational_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.operational).all() error_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.error).all() all_envs_last_recs = \ OpenStackWorkloadStatsCollection.get_last_by_resource_type( resource_type) ready_or_error_ids = set([c.id for c in operational_clusters] + [c.id for c in error_clusters]) envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \ ready_or_error_ids # Clear current resource data for unavailable clusters. # Current OSWL data is cleared for those clusters which status is not # 'operational' nor 'error' or when cluster was removed. Data is # cleared for cluster only if it was updated recently (today or # yesterday). While this collector is running with interval much # smaller than one day it should not miss any unavailable cluster. for id in envs_ids_to_clear: oswl_statistics_save(id, resource_type, []) # Collect current OSWL data and update data in DB for cluster in operational_clusters: try: client_provider = helpers.ClientProvider(cluster) proxy_for_os_api = utils.get_proxy_for_cluster(cluster) version_info = utils.get_version_info(cluster) with utils.set_proxy(proxy_for_os_api): data = helpers.get_info_from_os_resource_manager( client_provider, resource_type) oswl_statistics_save(cluster.id, resource_type, data, version_info=version_info) except errors.StatsException as e: logger.error("Cannot collect OSWL resource {0} for cluster " "with id {1}. Details: {2}." .format(resource_type, cluster.id, six.text_type(e)) ) except Exception as e: logger.exception("Error while collecting OSWL resource {0} " "for cluster with id {1}. Details: {2}." .format(resource_type, cluster.id, six.text_type(e)) ) db.commit() except Exception as e: logger.exception("Exception while collecting OS workloads " "for resource name {0}. Details: {1}" .format(resource_type, six.text_type(e))) finally: db.remove()
def make_ubuntu_preferences_task(uids, repo): # NOTE(ikalnitsky): In order to implement the proper pinning, # we have to download and parse the repo's "Release" file. # Generally, that's not a good idea to make some HTTP request # from Nailgun, but taking into account that this task # will be executed in uWSGI's mule worker we can skip this # rule, because proper pinning is more valuable thing right now. template = '\n'.join([ 'Package: *', 'Pin: release {conditions}', 'Pin-Priority: {priority}']) preferences_content = [] try: release = debian.get_release_file(repo, retries=3) release = debian.parse_release_file(release) pin = debian.get_apt_preferences_line(release) except requests.exceptions.HTTPError as exc: logger.error("Failed to fetch 'Release' file due to '%s'. " "The apt preferences won't be applied for repo '%s'.", six.text_type(exc), repo['name']) return None except Exception: logger.exception("Failed to parse 'Release' file.") return None # NOTE(kozhukalov): When a package is available both in: # 1) http://archive.ubuntu.com/ubuntu trusty universe # 2) http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0 mos7.0 main # And if the content of the preferences file is (i.e. by section priority): # Package: * # Pin: release o=Mirantis, a=mos7.0, n=mos7.0, l=mos7.0, c=main # Pin-Priority: 1050 # then the package available in MOS won't match the pin because for # some reason apt still thinks this package is in universe section. # As a result: # # apt-cache policy ohai # ohai: # Installed: (none) # Candidate: 6.14.0-2 # Version table: # 6.14.0-2 0 # 500 http://10.20.0.1/mirror/ubuntu/ trusty/universe amd64 Packages # 6.14.0-2~u14.04+mos1 0 # 500 http://10.20.0.2:8080/2015.1.0-7.0/ubuntu/x86_64/ mos7.0/main # amd64 Packages preferences_content.append(template.format( conditions=pin, priority=repo['priority'])) preferences_content = '\n\n'.join(preferences_content) preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name']) return make_upload_task(uids, preferences_content, preferences_path)
def content_json(func, cls, *args, **kwargs): json_resp = lambda data: (jsonutils.dumps(data) if isinstance( data, (dict, list)) or data is None else data) request_validate_needed = True response_validate_needed = True resource_type = "single" if issubclass(cls.__class__, CollectionHandler) and not func.func_name == "POST": resource_type = "collection" if (func.func_name in ("GET", "DELETE") or getattr(cls.__class__, 'validator', None) is None or resource_type == "single" and not cls.validator.single_schema or resource_type == "collection" and not cls.validator.collection_schema): request_validate_needed = False try: if request_validate_needed: BaseHandler.checked_data(cls.validator.validate_request, resource_type=resource_type) resp = func(cls, *args, **kwargs) except web.notmodified: raise except web.HTTPError as http_error: if http_error.status_code != 204: web.header('Content-Type', 'application/json', unique=True) if http_error.status_code >= 400: http_error.data = json_resp({ "message": http_error.data, "errors": http_error.err_list }) else: http_error.data = json_resp(http_error.data) raise # intercepting all errors to avoid huge HTML output except Exception as exc: logger.exception('Unexpected exception occured') http_error = BaseHandler.http( 500, (traceback.format_exc(exc) if settings.DEVELOPMENT else 'Unexpected exception, please check logs')) http_error.data = json_resp(http_error.data) web.header('Content-Type', 'text/plain') raise http_error if all([ settings.DEVELOPMENT, response_validate_needed, getattr(cls.__class__, 'validator', None) is not None ]): BaseHandler.checked_data(cls.validator.validate_response, resource_type=resource_type) web.header('Content-Type', 'application/json', unique=True) return json_resp(resp)
def PUT(self, cluster_id): """:returns: JSONized Task object. :http: * 202 (task successfully executed) * 400 (invalid object data specified) * 404 (environment is not found) * 409 (task with such parameters already exists) """ cluster = self.get_object_or_404( objects.Cluster, cluster_id, log_404=(u"warning", u"Error: there is no cluster " u"with id '{0}' in DB.".format(cluster_id))) logger.info(self.log_message.format(env_id=cluster_id)) try: options = self.get_options() except ValueError as e: raise self.http(400, six.text_type(e)) try: self.validator.validate(cluster) except errors.NailgunException as e: raise self.http(400, e.message) if objects.Release.is_lcm_supported(cluster.release): # try to get new graph to run transaction manager try: transaction_options = self.get_transaction_options( cluster, options) except errors.NailgunException as e: logger.exception("Failed to get transaction options") raise self.http(400, msg=six.text_type(e)) if transaction_options: return self.start_transaction(cluster, transaction_options) try: task_manager = self.task_manager(cluster_id=cluster.id) task = task_manager.execute(**options) except (errors.AlreadyExists, errors.StopAlreadyRunning) as exc: raise self.http(409, exc.message) except ( errors.DeploymentNotRunning, errors.NoDeploymentTasks, errors.WrongNodeStatus, errors.UnavailableRelease, errors.CannotBeStopped, ) as exc: raise self.http(400, exc.message) except Exception as exc: logger.error( self.log_error.format(env_id=cluster_id, error=str(exc))) # let it be 500 raise self.raise_task(task)
def collect(resource_type): try: operational_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.operational).all() error_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.error).all() all_envs_last_recs = \ OpenStackWorkloadStatsCollection.get_last_by_resource_type( resource_type) ready_or_error_ids = set([c.id for c in operational_clusters] + [c.id for c in error_clusters]) envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \ ready_or_error_ids # Clear current resource data for unavailable clusters. # Current OSWL data is cleared for those clusters which status is not # 'operational' nor 'error' or when cluster was removed. Data is # cleared for cluster only if it was updated recently (today or # yesterday). While this collector is running with interval much # smaller than one day it should not miss any unavailable cluster. for id in envs_ids_to_clear: oswl_statistics_save(id, resource_type, []) # Collect current OSWL data and update data in DB for cluster in operational_clusters: try: client_provider = helpers.ClientProvider(cluster) proxy_for_os_api = utils.get_proxy_for_cluster(cluster) version_info = utils.get_version_info(cluster) with utils.set_proxy(proxy_for_os_api): data = helpers.get_info_from_os_resource_manager( client_provider, resource_type) oswl_statistics_save(cluster.id, resource_type, data, version_info=version_info) except errors.StatsException as e: logger.error("Cannot collect OSWL resource {0} for cluster " "with id {1}. Details: {2}.".format( resource_type, cluster.id, six.text_type(e))) except Exception as e: logger.exception( "Error while collecting OSWL resource {0} " "for cluster with id {1}. Details: {2}.".format( resource_type, cluster.id, six.text_type(e))) db.commit() except Exception as e: logger.exception("Exception while collecting OS workloads " "for resource name {0}. Details: {1}".format( resource_type, six.text_type(e))) finally: db.remove()
def _distributed_serialize_tasks_for_node(formatter_contexts_idx, node_and_tasks, scattered_data): """Remote serialization call for DistributedProcessingPolicy Code of the function is copied to the workers and executed there, thus we are including all required imports inside the function. :param formatter_contexts_idx: dict of formatter contexts with node_id value as key :param node_and_tasks: list of node_id, task_data tuples :param scattered_data: feature object, that points to data copied to workers :return: [(node_id, serialized), error] """ try: factory = TasksSerializersFactory(scattered_data['context']) # Restoring settings settings.config = scattered_data['settings_config'] for k in formatter_contexts_idx: formatter_contexts_idx[k]['SETTINGS'] = settings except Exception as e: logger.exception("Distributed serialization failed") return [((None, None), e)] result = [] for node_and_task in node_and_tasks: node_id = None try: node_id, task = node_and_task logger.debug("Starting distributed node %s task %s serialization", node_id, task['id']) formatter_context = formatter_contexts_idx[node_id] serializer = factory.create_serializer(task) serialized = serializer.serialize( node_id, formatter_context=formatter_context) logger.debug( "Distributed node %s task %s serialization " "result: %s", node_id, task['id'], serialized) result.append(((node_id, serialized), None)) except Exception as e: logger.exception("Distributed serialization failed") result.append(((node_id, None), e)) break logger.debug("Processed tasks count: %s", len(result)) return result
def _distributed_serialize_tasks_for_node(formatter_contexts_idx, node_and_tasks, scattered_data): """Remote serialization call for DistributedProcessingPolicy Code of the function is copied to the workers and executed there, thus we are including all required imports inside the function. :param formatter_contexts_idx: dict of formatter contexts with node_id value as key :param node_and_tasks: list of node_id, task_data tuples :param scattered_data: feature object, that points to data copied to workers :return: [(node_id, serialized), error] """ try: factory = TasksSerializersFactory(scattered_data['context']) # Restoring settings settings.config = scattered_data['settings_config'] for k in formatter_contexts_idx: formatter_contexts_idx[k]['SETTINGS'] = settings except Exception as e: logger.exception("Distributed serialization failed") return [((None, None), e)] result = [] for node_and_task in node_and_tasks: node_id = None try: node_id, task = node_and_task logger.debug("Starting distributed node %s task %s serialization", node_id, task['id']) formatter_context = formatter_contexts_idx[node_id] serializer = factory.create_serializer(task) serialized = serializer.serialize( node_id, formatter_context=formatter_context) logger.debug("Distributed node %s task %s serialization " "result: %s", node_id, task['id'], serialized) result.append(((node_id, serialized), None)) except Exception as e: logger.exception("Distributed serialization failed") result.append(((node_id, None), e)) break logger.debug("Processed tasks count: %s", len(result)) return result
def _raise_error_task(self, cluster, task_name, exc): # set task status to error and update its corresponding data task = Task( name=task_name, cluster=cluster, status=consts.TASK_STATUSES.error, progress=100, message=six.text_type(exc) ) db().add(task) db().commit() logger.exception("Error in network configuration") self.raise_task(task)
def _serialize_task_for_node(factory, node_and_task): node_id, task = node_and_task logger.debug("applying task '%s' for node: %s", task['id'], node_id) try: task_serializer = factory.create_serializer(task) serialized = task_serializer.serialize(node_id) return node_id, serialized except Exception: logger.exception("failed to serialize task '%s' for node: %s", task['id'], node_id) raise
def must_send_stats(self): try: stat_settings = objects.MasterNodeSettings.get_one(). \ settings.get("statistics", {}) return stat_settings.get("user_choice_saved", {}).\ get("value", False) and \ stat_settings.get("send_anonymous_statistic", {}). \ get("value", False) except Exception as e: logger.exception( "Get statistics settings failed: %s", six.text_type(e)) return False
def must_send_stats(self): try: stat_settings = getattr(objects.MasterNodeSettings.get_one(), "settings", {}).get("statistics", {}) return stat_settings.get("user_choice_saved", {}).\ get("value", False) and \ stat_settings.get("send_anonymous_statistic", {}). \ get("value", False) except (AttributeError, TypeError) as e: logger.exception("Get statistics settings failed: %s", six.text_type(e)) return False
def ping_collector(self): try: resp = requests.get(settings.COLLECTOR_PING_URL, timeout=settings.COLLECTOR_RESP_TIMEOUT) resp.raise_for_status() return True except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException, requests.exceptions.HTTPError) as e: logger.exception("Collector ping failed: %s", six.text_type(e)) return False
def _raise_error_task(self, cluster, task_name, exc): # set task status to error and update its corresponding data task = Task(name=task_name, cluster=cluster, status=consts.TASK_STATUSES.error, progress=100, message=six.text_type(exc)) db().add(task) db().commit() logger.exception('Error in network configuration') self.raise_task(task)
def _handle_stats_opt_in(self): if self.single.must_send_stats(): logger.debug("Handling customer opt-in to sending statistics") manager = CreateStatsUserTaskManager() else: logger.debug("Handling customer opt-out to sending statistics") manager = RemoveStatsUserTaskManager() try: manager.execute() except Exception: logger.exception("Stats user operation failed")
def _serialize_task_for_node(factory, node_and_task): node_id, task = node_and_task logger.debug( "applying task '%s' for node: %s", task['id'], node_id ) try: task_serializer = factory.create_serializer(task) serialized = task_serializer.serialize(node_id) return node_id, serialized except Exception: logger.exception( "failed to serialize task '%s' for node: %s", task['id'], node_id ) raise
def set_default_node_volumes(cls, node): from .objects.volumes import VolumeObject try: VolumeObject.set_default_node_volumes(node) except Exception as exc: logger.exception(exc) msg = "Failed to generate volumes for node '{0}': '{1}'".format( node.human_readable_name, six.text_type(exc) ) Notification.create({"topic": "error", "message": msg, "node_id": node.id}) if node.cluster_id: Node.add_pending_change(node, "disks")
def ping_collector(self): try: resp = requests.get(self.build_collector_url("COLLECTOR_PING_URL"), timeout=settings.COLLECTOR_RESP_TIMEOUT) resp.raise_for_status() return True except (urllib3.exceptions.DecodeError, urllib3.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.TooManyRedirects, requests.exceptions.HTTPError) as e: logger.error("Collector ping failed: %s", type(e).__name__) except Exception as e: logger.exception("Collector ping failed: %s", six.text_type(e)) return False
def remove_silently(path): """Removes an element from file system no matter if it's file, folder or symlink. Ignores OSErrors. :param path: path """ try: if os.path.islink(path): os.unlink(path) elif os.path.isfile(path): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) except OSError as e: logger.exception(e)
def set_default_node_volumes(cls, node): from .objects.volumes import VolumeObject try: VolumeObject.set_default_node_volumes(node) except Exception as exc: logger.exception(exc) msg = "Failed to generate volumes for node '{0}': '{1}'".format( node.human_readable_name, six.text_type(exc)) objects.Notification.create({ 'topic': 'error', 'message': msg, 'node_id': node.id}) if node.cluster_id: objects.Node.add_pending_change(node, 'disks')
def set_proxy(self): if os.environ.get("http_proxy"): raise Exception("Cannot set 'http_proxy' environment variable " "as it already has a value") os.environ["http_proxy"] = self.proxy try: yield except Exception as e: logger.exception("Error while interacting with " "OpenStack api. Details: {0}".format( six.text_type(e))) finally: if os.environ.get("http_proxy") == self.proxy: del (os.environ["http_proxy"])
def delete_expired_oswl_entries(): try: deleted_rows_count = \ objects.OpenStackWorkloadStatsCollection.clean_expired_entries() if deleted_rows_count == 0: logger.info("There are no expired OSWL entries in db.") db().commit() logger.info("Expired OSWL entries are " "successfully cleaned from db") except Exception as e: logger.exception("Exception while cleaning oswls entries from " "db. Details: {0}".format(six.text_type(e))) finally: db.remove()
def prepare_action_log_kwargs(cls, task): """Prepares kwargs dict for ActionLog db model class :param task: task instance to be processed :returns: kwargs dict for action log creation """ create_kwargs = { 'task_uuid': task.uuid, 'cluster_id': task.cluster_id, 'action_group': tasks_names_actions_groups_mapping[task.name], 'action_name': task.name, 'action_type': consts.ACTION_TYPES.nailgun_task, 'start_timestamp': datetime.datetime.utcnow() } # actor_id passed from ConnectionMonitor middleware and is # needed for binding task execution event with particular # actor actor_id = None try: actor_id_field = 'fuel.action.actor_id' if hasattr(web.ctx, 'env') and actor_id_field in web.ctx.env: # Fetching actor_id from env context actor_id = web.ctx.env.get(actor_id_field) else: # Fetching actor_id from parent task action log from nailgun import objects # preventing cycle import error if task.parent_id: parent_task = objects.Task.get_by_uid(task.parent_id) action_log = objects.ActionLog.get_by_kwargs( task_uuid=parent_task.uuid, action_name=parent_task.name) actor_id = action_log.actor_id except Exception: logger.exception("Extracting of actor_id failed") create_kwargs['actor_id'] = actor_id additional_info = { 'parent_task_id': task.parent_id, 'subtasks_ids': [t.id for t in task.subtasks], 'operation': task.name } create_kwargs['additional_info'] = additional_info return create_kwargs
def send_serialized(records): if records: logger.info("Send %d records", len(records)) try: req_body = {"action_logs": records} headers = {'content-type': 'application/json'} resp = requests.post( settings.COLLECTOR_STORE_URL, headers=headers, data=jsonutils.dumps(req_body), timeout=settings.COLLECTOR_RESP_TIMEOUT) resp.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException, requests.exceptions.HTTPError) as e: logger.exception( "Action logs sending to collector failed: %s", six.text_type(e)) else: resp_dict = resp.json() if resp.status_code == requests.codes.created and \ resp_dict["status"] == \ consts.LOG_CHUNK_SEND_STATUS.ok: records_resp = resp_dict["action_logs"] saved_ids = set() failed_ids = set() for record in records_resp: if record["status"] == \ consts.LOG_RECORD_SEND_STATUS.failed: failed_ids.add(record["external_id"]) else: saved_ids.add(record["external_id"]) sent_saved_ids = set(saved_ids) & set(ids) logger.info("Records saved: %s, failed: %s", six.text_type(list(sent_saved_ids)), six.text_type(list(failed_ids))) db().query(models.ActionLog).filter( models.ActionLog.id.in_(sent_saved_ids) ).update( {"is_sent": True}, synchronize_session=False ) db().commit() else: logger.error("Unexpected collector answer: %s", six.text_type(resp))
def send_data_to_url(self, url, data): resp = None try: headers = {"content-type": "application/json", "master-node-uid": InstallationInfo().get_master_node_uid()} resp = requests.post( url, headers=headers, data=jsonutils.dumps(data), timeout=settings.COLLECTOR_RESP_TIMEOUT ) except ( urllib3.exceptions.DecodeError, urllib3.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.TooManyRedirects, ) as e: logger.error("Sending data to collector failed: %s", type(e).__name__) except Exception as e: logger.exception("Sending data to collector failed: %s", six.text_type(e)) return resp
def send_data_to_url(self, url, data): resp = None try: headers = {'content-type': 'application/json'} resp = requests.post(url, headers=headers, data=jsonutils.dumps(data), timeout=settings.COLLECTOR_RESP_TIMEOUT) except (urllib3.exceptions.DecodeError, urllib3.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.TooManyRedirects) as e: logger.error("Sending data to collector failed: %s", type(e).__name__) except Exception as e: logger.exception("Sending data to collector failed: %s", six.text_type(e)) return resp
def GET(self, cluster_id): """:returns: JSONized network configuration for cluster. :http: * 200 (OK) * 404 (cluster not found in db) """ cluster = self.get_object_or_404(objects.Cluster, cluster_id) self.check_net_provider(cluster) try: # there are a plenty of reasons why serializer could throw # an exception. usually that means we don't handle properly # some corner cases, and it should be fixed. in order # to simplify troubleshootng, let's print traceback to log. return self.serializer.serialize_for_cluster(cluster) except Exception: logger.exception('Serialization failed') raise
def set_proxy(self): if os.environ.get("http_proxy"): raise Exception( "Cannot set 'http_proxy' environment variable " "as it already has a value" ) os.environ["http_proxy"] = self.proxy try: yield except Exception as e: logger.exception("Error while interacting with " "OpenStack api. Details: {0}" .format(six.text_type(e))) finally: if os.environ.get("http_proxy") == self.proxy: del(os.environ["http_proxy"])
def get_openstack_info(self, cluster, cluster_nodes): if not cluster.status == consts.CLUSTER_STATUSES.operational: return {} info = {} try: getter = \ openstack_info_collector.OpenStackInfoCollector( cluster, cluster_nodes ) info = getter.get_info() except Exception as e: logger.exception("Cannot collect OpenStack info due to error: %s", six.text_type(e)) return info
def get_version_info(cluster): """Returns current Fuel and OpenStack version info :param cluster: cluster :type cluster: nailgun.db.sqlalchemy.models.Cluster :return: dict with version info or None """ try: return { 'fuel_version': cluster.fuel_version, 'release_version': cluster.release.version, 'release_name': cluster.release.name, 'release_os': cluster.release.operating_system, 'environment_version': cluster.release.environment_version } except Exception: logger.exception("Fetching version info for cluster '%s' failed", cluster)
def _handle_stats_opt_in(self, settings_data=None): """Starts task on stats user creation or removal :param settings_data: dict with master node settings. Current data from DB will be used if master_node_settings_data is None """ must_send = self.single.must_send_stats( master_node_settings_data=settings_data) if must_send: logger.debug("Handling customer opt-in to sending statistics") manager = CreateStatsUserTaskManager() else: logger.debug("Handling customer opt-out to sending statistics") manager = RemoveStatsUserTaskManager() try: manager.execute() except Exception: logger.exception("Stats user operation failed")
def _execute_async(self, supertask_id): logger.info(u"ApplyChangesTask: execute async starting for task %s", supertask_id) supertask = objects.Task.get_by_uid(supertask_id) try: self._execute_async_content(supertask) except Exception as e: logger.exception('Error occurred when running task') data = { 'status': consts.TASK_STATUSES.error, 'progress': 100, 'message': u'Error occurred when running task: {0}'.format( e.message), } objects.Task.update(supertask, data) db().commit()
def make_ubuntu_preferences_task(uids, repo): # NOTE(ikalnitsky): In order to implement the proper pinning, # we have to download and parse the repo's "Release" file. # Generally, that's not a good idea to make some HTTP request # from Nailgun, but taking into account that this task # will be executed in uWSGI's mule worker we can skip this # rule, because proper pinning is more valuable thing right now. template = '\n'.join([ 'Package: *', 'Pin: release {conditions}', 'Pin-Priority: {priority}']) preferences_content = [] try: release = debian.get_release_file(repo, retries=3) release = debian.parse_release_file(release) pin = debian.get_apt_preferences_line(release) except requests.exceptions.HTTPError as exc: logger.error("Failed to fetch 'Release' file due to '%s'. " "The apt preferences won't be applied for repo '%s'.", six.text_type(exc), repo['name']) return None except Exception: logger.exception("Failed to parse 'Release' file.") return None # if sections are detected (non-flat repo) create pinning per # sections; otherwise - create just one pin for the entire repo if repo['section']: for section in repo['section'].split(): preferences_content.append(template.format( conditions='{0},c={1}'.format(pin, section), priority=repo['priority'])) else: preferences_content.append(template.format( conditions=pin, priority=repo['priority'])) preferences_content = '\n\n'.join(preferences_content) preferences_path = '/etc/apt/preferences.d/{0}.pref'.format(repo['name']) return make_upload_task(uids, preferences_content, preferences_path)
def get_openstack_info(self, cluster, cluster_nodes): if not cluster.status == consts.CLUSTER_STATUSES.operational: return {} info = {} try: getter = \ openstack_info_collector.OpenStackInfoCollector( cluster, cluster_nodes ) info = getter.get_info() except Exception as e: logger.exception( "Cannot collect OpenStack info due to error: %s", six.text_type(e) ) return info
def collect(resource_type): try: operational_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.operational).all() for cluster in operational_clusters: client_provider = utils.ClientProvider(cluster) proxy_for_os_api = utils.get_proxy_for_cluster(cluster) with utils.set_proxy(proxy_for_os_api): data = utils.get_info_from_os_resource_manager( client_provider, resource_type) oswl_statistics_save(cluster.id, resource_type, data) db.commit() except Exception as e: logger.exception("Exception while collecting OS workloads " "for resource name {0}. Details: {1}".format( resource_type, six.text_type(e))) finally: db.remove()
def _execute_async(self, supertask_id): """Function for execute task in the mule :param supertask_id: id of parent task """ logger.info(u"ApplyChangesTask: execute async starting for task %s", supertask_id) supertask = objects.Task.get_by_uid(supertask_id) try: self._execute_async_content(supertask) except Exception as e: logger.exception('Error occurred when running task') data = { 'status': consts.TASK_STATUSES.error, 'progress': 100, 'message': u'Error occurred when running task: {0}'.format( e.message), } objects.Task.update(supertask, data) db().commit()
def _call_silently(self, task, instance, *args, **kwargs): # create action_log for task al = TaskHelper.create_action_log(task) method = getattr(instance, kwargs.pop('method_name', 'execute')) if task.status == consts.TASK_STATUSES.error: TaskHelper.update_action_log(task, al) return try: to_return = method(task, *args, **kwargs) # update action_log instance for task # for asynchronous task it will be not final update # as they also are updated in rpc receiver TaskHelper.update_action_log(task, al) return to_return except errors.NoChanges as e: self._finish_task(task, al, consts.TASK_STATUSES.ready, str(e)) except Exception as exc: logger.exception("Task '%s' failed.", task.name) self._finish_task(task, al, consts.TASK_STATUSES.error, str(exc))
def process_task(self, task, node_ids): """Processes one task one nodes of cluster. :param task: the task instance :param node_ids: the list of nodes, where this tasks should run """ logger.debug("applying task '%s' for nodes: %s", task['id'], node_ids) task_serializer = self.factory.create_serializer(task) for node_id in node_ids: try: task = task_serializer.serialize(node_id) except Exception: logger.exception("Failed to serialize task %s", task['id']) raise node_tasks = self.tasks_graph.setdefault(node_id, {}) # de-duplication the tasks on node # since task can be added after expand group need to # overwrite if existed task is skipped and new is not skipped. if self.need_update_task(node_tasks, task): node_tasks[task['id']] = task
def handle_task(self, cluster, **kwargs): if objects.Release.is_lcm_supported(cluster.release): # this code is actual only if cluster is LCM ready try: transaction_options = self.get_transaction_options( cluster, kwargs) except errors.NailgunException as e: logger.exception("Failed to get transaction options.") raise self.http(400, six.text_type(e)) if transaction_options: return self.start_transaction(cluster, transaction_options) nodes = self.get_nodes(cluster) try: task_manager = self.task_manager(cluster_id=cluster.id) task = task_manager.execute(nodes, **kwargs) except Exception as exc: logger.exception(u'Cannot execute %s task nodes: %s', self.task_manager.__name__, ','.join(n.uid for n in nodes)) raise self.http(400, msg=six.text_type(exc)) self.raise_task(task)
def on_load_failure(manager, endpoint, exc): logger.exception("Failed to load %s extension", endpoint.name) raise
def content_json(func, cls, *args, **kwargs): json_resp = lambda data: ( jsonutils.dumps(data) if isinstance(data, (dict, list)) or data is None else data ) request_validate_needed = True response_validate_needed = True resource_type = "single" if issubclass( cls.__class__, CollectionHandler ) and not func.func_name == "POST": resource_type = "collection" if ( func.func_name in ("GET", "DELETE") or getattr(cls.__class__, 'validator', None) is None or resource_type == "single" and not cls.validator.single_schema or resource_type == "collection" and not cls.validator.collection_schema ): request_validate_needed = False try: if request_validate_needed: BaseHandler.checked_data( cls.validator.validate_request, resource_type=resource_type ) resp = func(cls, *args, **kwargs) except web.notmodified: raise except web.HTTPError as http_error: if http_error.status_code != 204: web.header('Content-Type', 'application/json', unique=True) if http_error.status_code >= 400: http_error.data = json_resp({ "message": http_error.data, "errors": http_error.err_list }) else: http_error.data = json_resp(http_error.data) raise # intercepting all errors to avoid huge HTML output except Exception as exc: logger.exception('Unexpected exception occured') http_error = BaseHandler.http( 500, ( traceback.format_exc(exc) if settings.DEVELOPMENT else six.text_type(exc) ) ) http_error.data = json_resp(http_error.data) raise http_error if all([ settings.DEVELOPMENT, response_validate_needed, getattr(cls.__class__, 'validator', None) is not None ]): BaseHandler.checked_data( cls.validator.validate_response, resource_type=resource_type ) web.header('Content-Type', 'application/json', unique=True) return json_resp(resp)