def _update_command_log_level(self, module, log_level, splunk_service): logger.debug( "action=try_to_update_command_log_level module=%s, level=%s" % (module, log_level)) command_endpoint = "configs/conf-commands/%s" # use this property to configure log level log_level_property = "-DDBX_COMMAND_LOG_LEVEL=" entity = client.Entity(splunk_service, command_endpoint % module) to_update = { key: value for key, value in entity.content.items() if log_level_property in value } if len(to_update) != 1: raise Exception( "%s stanza in commands conf file is not valid because there must exist" " one and only one attribute with a value -DDBX_COMMAND_LOG_LEVEL=${LOG_LEVEL}", module) log_args = to_update.keys()[0] log_level_value = to_update[log_args] original_log_level = log_level_value.split(log_level_property)[1] if original_log_level != log_level: logger.debug( "action=update_command_log_level module=%s, original_level=%s level=%s" % (module, original_log_level, log_level)) to_update[log_args] = log_level_property + log_level entity.update(**to_update).refresh()
def handle_POST(self): try: pre_taskserverport = self.read_taskserver_port() payload = loads(self.request['payload']) self.check_java_home(payload) # check whether the javaHome is valid self.validate_java_home(payload["javaHome"]) self.update_vmopts(payload) splunk_service = SplunkServiceFactory.create(self.sessionKey, app='splunk_app_db_connect', owner=self.userName) entity = client.Entity(splunk_service, self.endpoint) entity.update(**payload).refresh() logger.debug('updated java settings') self.update_dbx_java_home(payload["javaHome"]) self.reset_java_command_filename(splunk_service) self.read_vmopts(entity.content) self.restart_task_server(pre_taskserverport) self.writeJson(entity.content) except Exception as ex: self.response.setStatus(500) self.writeJson({ "code": 500, "message": ex.message, "detail": str(ex) })
def write(jvm_options): jvm_options_file_path = _get_jvm_options_filepath() with open(jvm_options_file_path, "w") as vmopts_file: logger.debug( "action=write_jvm_options_to_file, filepath=%sjvmopts=%s" % (jvm_options_file_path, jvm_options)) vmopts_file.write(jvm_options)
def reset_java_command_filename(self, splunk_service): for java_command in self.java_commands: entity = client.Entity(splunk_service, self.commands_endpoint % java_command) # If customer have set the filename to "customized.java.path", we need to reset it to "java.path" # Related issue: DBX-3746 if entity["filename"] == self.customized_java_path: entity.update(filename="java.path").refresh() logger.debug("action=reset_java_command_filename command=%s" % java_command)
def _is_enterprise_product(self): server_info = self.service.info() product_type = server_info['product_type'] is_enterprise = product_type == 'enterprise' logger.debug( 'action=test_if_enterprise_product product_type=%s result=%s', product_type, is_enterprise) return is_enterprise
def set_property(jvmopts, property, property_regex, value): prefix = " -D" if property in jvmopts: logger.debug("action=replace_property_value_in_jvm_options, property=%s value=%s property_regex=%s" %(property, value, property_regex)) jvmopts= re.sub(property_regex, property + "=" + value, jvmopts) else: logger.debug("action=append_property_value_in_jvm_options, property=%s value=%s" %(property, value)) jvmopts += (prefix + property + "=" + value) return jvmopts
def is_clustering_enabled(self): # DBX-1741 - don't check for clustering if we are not on enterprise is_enterprise = self._is_enterprise_product() if is_enterprise is False: return False mode = self.content['mode'] enabled = is_enterprise and mode != 'disabled' logger.debug( 'action=retrieve_shc_clustering clustering_mode=%s clustering_enabled=%s', mode, enabled) return enabled
def _test_java_in_path(): try: p = subprocess.Popen(['which', 'java'], stdout=subprocess.PIPE) out = p.communicate()[0] if out: path = os.path.normpath(os.path.join(os.path.realpath(out.strip()), '..', '..')) logger.debug("'which java' returned %s", path) v = _get_java_version(path) if v: return path except: pass
def handler_wrapper(url, message, **kwargs): logger.debug('action=sending_request url=%s message=%s kwargs=%s', url, message, kwargs) try: return custom_handler(url, message, **kwargs) except ssl.SSLError: message = 'Unable to communicate with Splunkd. ' \ 'If you enable requireClientCert please make sure certs folder contains privkey.pem and cert.pem files. ' \ 'Also make sure cert.pem has been signed by the root CA used by Splunkd.' logger.error(message, exc_info=True) raise ssl.SSLError(message)
def get_private_key(key_path=None): ''' load local instance private key and return rsa instance ''' try: if (key_path is None): key_path = PRIV_KEY_PATH logger.debug("Loading private key from: %s" % os.path.expandvars(key_path)) return RSA.load_key(key_path) except Exception, ex: logger.error('unable to load private key %s' % key_path) logger.debug(ex) return None
def get_property(jvmopts, property, property_regex): vmopts = jvmopts.split(property) if len(vmopts) > 2: raise Exception( 'failed to parse vmopts: too many property [%s] have been set' %property) matched = re.match(r'.*' + property_regex + r'.*', jvmopts) if matched: logger.debug("action=read_property_from_jvmoptions, jvmoptions=%s property=%s property_regex=%s" % (jvmopts, property, property_regex)) return matched.group(1) else: logger.debug("action=fail_to_read_property_from_jvmoptions, jvmoptions=%s property=%s property_regex=%s" % (jvmopts, property, property_regex)) return None
def check_response_integrity(self, headers, resp_status, resp_body): # just in case, others respond to the request if 'Server' not in headers or headers['Server'] != 'DBX Server': resp_status = (requests.codes.bad_gateway, 'Response was not generated by DBX Server, ' 'please make sure it is started and listening on %s port ' 'or consult documentation for details' % self.taskserverPort) logger.debug("action=check_response_integrity error=%s", resp_status[1]) resp_body = json.dumps({ 'code': resp_status[0], 'message': 'Bad Gateway', 'detail': resp_status[1] }) return resp_status, resp_body
def _update_server_log_level_in_file(self, server_log_level): logger.debug("action=try_to_update_server_log_level %s" % str(server_log_level)) try: jvmopts = jvm_options.read() logger.debug("action=read_vmopts_from_file vmopts=%s" % jvmopts) for (module, level) in server_log_level.items(): property = self.server_module_info[module]["property"] regex = self.server_module_info[module]["regex"] jvmopts = jvm_options.set_property(jvmopts, property, regex, level) jvm_options.write(jvmopts) except Exception as ex: logger.error('unable to update vmopts file [%s]' % ex) raise
def _check_java_server_configuration_health(self): # check if server.jar exists final_status = Status.OK final_details = [] logger.info("check the existence of server.jar") path_to_server_jar = os.path.join(self.path_to_app, "jars", "server.jar") if not os.path.isfile(path_to_server_jar): detail = "{} does not exist".format(path_to_server_jar) final_status = Status.ERROR final_details.append(detail) return self._create_health_check_result(final_status, final_details) # check whether it can be successfully registered to Splunk logger.info( "check whether the [server://default] modular input is registered successfully" ) input_url = "https://{}/services/data/modular-inputs/server".format( self.hostPath) try: response = requests.get( url=input_url, headers={'Authorization': ('Splunk %s' % self.sessionKey)}, verify=False) if response.status_code != 200: final_status = Status.ERROR final_details.append( "The modular input to start the Java Server failed to be registered to Splunk" ) return self._create_health_check_result( final_status, final_details) except Exception as ex: logger.error( "action=fail_to_send_request_to_get_health_check_result", ex) final_status = Status.ERROR final_details.append( "The modular input to start the Java Server failed to be registered to Splunk" ) return self._create_health_check_result(final_status, final_details) # check the availability of java server logger.debug("check the availability of java server") status, result = self._check_java_server_availability() if status != Status.OK: return self._create_health_check_result(status, result) return self._create_health_check_result(final_status, final_details)
def update_vmopts(self, content): try: jvmopts = content.pop('jvmOptions', '') # jvmOptions may contain taskServerPort settings taskServerPort = content.pop('taskServerPort', self.defaultPort) logger.debug('action=get_vmopts_from_postdata, jvmOptions: [%s], taskServerPort: [%s]' % (jvmopts, taskServerPort)) if not isinstance(taskServerPort, int): raise Exception("task server port must be a int value") if taskServerPort < 1024 or taskServerPort > 65535: raise Exception('task server port must be a number in [1024, 65535]') jvmopts = jvm_options.set_property(jvmopts, self.taskserverPortProperty, self.taskserverPortRegex, str(taskServerPort)) jvm_options.write(jvmopts) except Exception as ex: logger.error('unable to update vmopts file [%s]' % ex) raise
def read(): try: jvm_options_file_path = _get_jvm_options_filepath() if not os.path.isfile(jvm_options_file_path): logger.debug("action=jvm_options_file_not_exist, file_path=%s" % jvm_options_file_path) return "" with open(jvm_options_file_path, 'r') as vmopts_file: vmopts = vmopts_file.readline().strip() if not vmopts: return "" else: return vmopts except Exception as ex: logger.warn("action=fail_to_read_jvm_options_from_file", ex) return ""
def _check_java_installation_health(self): platform = self._detect_platform_dir() path_to_customized_java = os.path.join(self.path_to_app, platform, "bin", "customized.java.path") final_status = Status.OK final_details = [] if not os.path.isfile(path_to_customized_java): # check if $JAVA_HOME/bin/java satisfies the requirement of JAVA 8 if JAVA_HOME is defined java_home = os.getenv("JAVA_HOME") if java_home: logger.debug("JAVA_HOME is {}".format(java_home)) java_cmd = os.path.join(java_home, "bin", "java") else: logger.debug( "JAVA_HOME is not specified. Use default java command.") java_cmd = "java" # validating java command is_healthy, detail = validateJRE(java_cmd) if is_healthy: splunk_service = SplunkServiceFactory.create( self.sessionKey, app='splunk_app_db_connect', owner=self.userName) result = self._check_commands(splunk_service) result.append( self._check_modular_alert_java_conf(splunk_service)) # we reduce the results for status, details in result: if status != Status.OK: final_status = Status.ERROR final_details.append(details) else: final_status = Status.ERROR final_details.append(detail) else: # check the content of customized.java.path with open(path_to_customized_java, 'r') as customized_java_file: java_cmd = customized_java_file.readline().strip() is_healthy, final_details = validateJRE(java_cmd) if not is_healthy: final_status = Status.ERROR return self._create_health_check_result(final_status, final_details)
def getRemoteKey(node, sessionKey): parsed_url = urlparse.urlparse(node) url = parsed_url.scheme + "://" + parsed_url.netloc ''' for local calls we should user the current session ''' local_settings = en.getEntity('/server/settings', 'settings', sessionKey=sessionKey) local_url = "https://" + local_settings["host"] + ":" + local_settings[ "mgmtHostPort"] if (url.lower() == local_url.lower()): return sessionKey '''TODO: change this to splunkds endpoint''' local_host = get_local_host(None, sessionKey) remote_user = local_user = "******" remoteKey = get_remote_token(url, remote_user, local_host, local_user) logger.debug("getRemoteKey: got remote token for: %s %s %s %s %s" % (url, remote_user, local_host, local_user, remoteKey)) return remoteKey
def is_captain(self, retryWait=2, maxRetries=6): retry = True step = 2 retries = 0 # exponential backoff because of SPL-90689 while retry and retries < maxRetries: try: self.refresh() captain = self.content local_settings = ServerSettings(self.service) parsed_peer_scheme = urlparse.urlparse( captain['peer_scheme_host_port']) #dns comparisons are lowercase, https://tools.ietf.org/html/rfc4343 local_server_name = local_settings['serverName'].lower() remote_server_name = captain['label'].lower() #until Ember we compare /server/settings/serverName vs shcluster/captain/info/label #and /server/settings/mgmtHostPort vs port in shcluster/captain/info/peer_scheme_host_port #for Ember, we have to use noProxy, see DBX-1774 if local_server_name != remote_server_name: logger.debug( 'action=not_shc_captain cause=server_name_unmatched local_name=%s remote_name=%s', local_server_name, remote_server_name) return False local_server_port = int(local_settings['mgmtHostPort']) remote_server_port = int(parsed_peer_scheme.port) if local_server_port != remote_server_port: logger.debug( 'action=not_shc_captain cause=port_unmatched local_port=%s remote_port=%s', local_server_port, remote_server_port) return False return True except HTTPError as he: if he.status == 503: retry = True time.sleep(retryWait) retryWait = retryWait * step retries = retries + 1 else: return False logger.info('action=unable_to_determine_if_running_on_captain') return False
def _update_server_log_level_runtime(self, server_log_level): data = [] for (module, level) in server_log_level.items(): packages = self.server_module_info[module]["packages"] for package in packages: data.append({ "logger": package, "level": level }) try: logger.debug("action=send_log_level_update_request url=%s data=%s" % (self.server_log_level_url, dumps(data))) headers = { 'content-type': 'application/json' } response = requests.put(url=self.server_log_level_url, headers=headers, data=dumps(data), verify=False) if response.status_code != 200: logger.warn("acton=fail_to_send_request_to_update_log_level status=%s content=%s" % (response.status_code, response.content)) except Exception as ex: # TODO: whether to raise error? vmopts file have been updated successfully logger.warn("acton=fail_to_send_request_to_update_log_level", ex)
def get_token(url, localSessionKey): parsed_url = urlparse.urlparse(url) remote_host = parsed_url.scheme + "://" + parsed_url.netloc #DBX-2442 if not utils.validateHostName(remote_host): logger.debug("invalid remote_host") return None ''' for local calls we should user the current session ''' local_settings = en.getEntity('/server/settings', 'settings', sessionKey=localSessionKey) local_url = local_settings["host"] + ":" + local_settings["mgmtHostPort"] if (parsed_url.netloc.lower() == local_url.lower()): return localSessionKey local_host = get_local_host(None, localSessionKey) remote_user = local_user = "******" remoteKey = get_remote_token(remote_host, remote_user, local_host, local_user) logger.debug("getRemoteKey: got remote token for: %s %s %s %s %s" % (remote_host, remote_user, local_host, local_user, remoteKey)) return remoteKey
def set_certificate(remote_host, local_host, local_cert, sessionKey): #DBX-2442 if not utils.validateHostName(remote_host): raise Exception("invalid remote_host") if not utils.validateHostName(local_host): raise Exception("invalid local_host") postargs = {PEERNAME: local_host, CERTIFICATE: local_cert} logger.debug('set_certificate: remote_host=%s postargs=%s' % (remote_host, postargs)) resp, cont = rest.simpleRequest(remote_host + AUTH_CERT_ENDPOINT + '/' + local_host, sessionKey=sessionKey, postargs=postargs) if resp.status not in [200, 201]: logger.error('unable to post certificate to remote peer %s %d' % (remote_host, resp.status)) raise Exception('posting certificate to remote peer failed') return
def _check_file_permission_health(self): unhealthy_folders = [] paths_to_check_permission = [ self.path_to_app, self.path_to_checkpoint, self.path_to_log ] for path in paths_to_check_permission: # TODO: do we need to check sub dir permission? has_read_and_write_permission = (os.access(path, os.R_OK) and os.access(path, os.W_OK)) logger.debug("file: {}, has read/write permission: {}".format( path, has_read_and_write_permission)) if not has_read_and_write_permission: unhealthy_folders.append(path) if unhealthy_folders: status = Status.ERROR detail = "Following folder(s): {} don't exist or don't have read/write permissions".format( " ".join(unhealthy_folders)) else: status = Status.OK detail = "All folders have appropriate permissions" return [self._create_health_check_result(status, [detail])]
def _create_handler(cls): script_dir = os.path.dirname(os.path.realpath(__file__)) key_file_location = os.path.join(script_dir, '..', '..', '..', 'certs', 'privkey.pem') cert_file_location = os.path.join(script_dir, '..', '..', '..', 'certs', 'cert.pem') key_file_exists = os.path.isfile(key_file_location) cert_file_exists = os.path.isfile(cert_file_location) if key_file_exists and cert_file_exists: logger.debug('action=enable_client_certicates') else: key_file_location = cert_file_location = None if key_file_exists != cert_file_exists: logger.warn( 'Unable to enable client certificate because the key or the certificate is missing.' \ 'certs application folder should contain privkey.pem and cert.pem files') custom_handler = handler(key_file=key_file_location, cert_file=cert_file_location) # we wrap the handler to intercept any SSL exception to give a meaningful message to end user def handler_wrapper(url, message, **kwargs): logger.debug('action=sending_request url=%s message=%s kwargs=%s', url, message, kwargs) try: return custom_handler(url, message, **kwargs) except ssl.SSLError: message = 'Unable to communicate with Splunkd. ' \ 'If you enable requireClientCert please make sure certs folder contains privkey.pem and cert.pem files. ' \ 'Also make sure cert.pem has been signed by the root CA used by Splunkd.' logger.error(message, exc_info=True) raise ssl.SSLError(message) return handler_wrapper
def forwardRequest(self, method, url, params=None, data=None): headers = { 'content-type': 'application/json', 'X-DBX-SESSION_KEY': self.sessionKey, 'X-DBX-OWNER': self.userName } resp_status = () resp_body = {} if data is not None: data = data.encode('utf-8') try: logger.debug( "action=api_forwarding_request method=%s url=%s params=%s timeout=%s", method, url, params, self.timeout) response = requests.request(method=method, url=url, data=data, params=params, headers=headers, timeout=self.timeout) resp_status = (response.status_code, '') resp_body = response.content resp_status, resp_body = self.check_response_integrity( response.headers, resp_status, resp_body) except Exception as e: resp_status = self.handle_exception(e) resp_body = json.dumps({ 'code': resp_status[0], 'message': resp_status[1], 'detail': str(e) }) logger.debug("action=api_forwarding_request_error error=%s", e) finally: # Make sure a JSON response is returned logger.debug( "action=api_forwarding_request_writing_response status=%s", resp_status[0]) self.response.setHeader('content-type', 'application/json') self.response.status = resp_status[0] self.response.write(resp_body)
def call_peer(remote_host, remote_user, local_host, local_user, ts, nonce, signature): #DBX-2442 if not utils.validateHostName(remote_host): logger.debug("invalid remote_host") return None if not utils.validateHostName(local_host): logger.debug("invalid local_host") return None if not utils.validateUserName(remote_user): logger.debug("invalid remote_user") return None if not utils.validateUserName(local_user): logger.debug("invalid local_user") return None ''' make remote REST call to retreive foreign sessionKey ''' postargs = { 'name': '_create', USERID: local_user, PEERNAME: local_host, USERNAME: remote_user, TS: ts, NONCE: nonce, SIGNATURE: signature } logger.debug('call peer: remote_host=%s postargs=%s' % (remote_host, postargs)) resp, cont = rest.simpleRequest(remote_host + AUTH_TOKENS_ENDPOINT, postargs=postargs) if resp.status not in [200, 201]: logger.error('unable to get session key from remote peer %s' % remote_host) return None try: atomEntry = rest.format.parseFeedDocument(cont) logger.debug('response from peer:\n%s' % atomEntry.toPrimitive()) ret = atomEntry.toPrimitive()[remote_user][remote_user] return ret except Exception, ex: logger.error('unable to parse response from remote peer %s' % remote_host) logger.exception(ex) return None