def validate_arguments(): val_data = get_validation_data() if 'debugMode' in val_data: debugMode = val_data['debugMode'].lower() if debugMode == "true" or debugMode == "t": logging.getLogger().setLevel(logging.DEBUG) useLocalAuth = False if 'useLocalAuth' in val_data: useLocalAuth = val_data['useLocalAuth'].lower() if useLocalAuth == "true" or useLocalAuth == "t": useLocalAuth = True logger.debug("useLocalAuth enabled") if val_data['destURL'] != "https://*****:*****@") logger.debug( "Attempting to replace proxy=%s by subsituting=%s with a password" % (proxies['https'], proxies['https'][start:end])) temp_password = get_password(proxies['https'][start:end], session_key, logger) proxies['https'] = proxies['https'][ 0:start - 9] + temp_password + proxies['https'][end:] try: logger.debug( "Running query against URL %s with username %s proxies_length=%s" % (url, destUsername, len(proxies), sslVerify)) res = requests.get(url, auth=(destUsername, destPassword), verify=sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, destUsername)) if (res.status_code != requests.codes.ok): print_error( "Attempt to validate access to Splunk failed with code %s, reason %s, text %s on URL %s" % (res.status_code, res.reason, res.text, url)) sys.exit(1) except requests.exceptions.RequestException as e: print_error( "Attempt to validate access to Splunk failed with error %s" % (e)) sys.exit(1) gitRepoURL = val_data['gitRepoURL'] if 'git_command' in val_data: git_command = val_data['git_command'].strip() logger.debug("Overriding git command to %s" % (git_command)) else: git_command = "git" if 'ssh_command' in val_data: ssh_command = val_data['ssh_command'].strip() logger.debug("Overriding ssh command to %s" % (ssh_command)) else: ssh_command = "ssh" (stdout, stderr, res) = runOSProcess("%s ls-remote %s" % (git_command, gitRepoURL), logger) #If we didn't manage to ls-remote perhaps we just need to trust the fingerprint / this is the first run? if res == False: (stdout, stderrout, res) = runOSProcess( ssh_command + " -n -o \"BatchMode yes\" -o StrictHostKeyChecking=no " + gitRepoURL[:gitRepoURL.find(":")], logger) (stdout, stderr, res) = runOSProcess("%s ls-remote %s" % (git_command), logger) if res == False: print_error( "Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr)) sys.exit(1)
def handle_POST(self): starttime = calendar.timegm(time.gmtime()) payload = six.moves.urllib.parse.parse_qs(self.request['payload']) #self.response.write(str(payload) + "\n") #currently we only receive the Splunk authorization key, so obtain that req_attributes = [ 'Authorization', 'splunk_vc_name', 'app', 'type', 'obj_name', 'tag', 'scope', 'timeout' ] for attr in req_attributes: if attr not in payload: #Don't log the authorization key if it was sent if 'Authorization' in payload: del payload['Authorization'] logger.error( "Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload)) self.response.write( "Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload)) return splunk_vc_name = payload['splunk_vc_name'][0] #self.response.write(splunk_vc_name + "\n\n\n") #Now run queries locally to check if the mentioned config matches an existing backup name headers = {"Authorization": "Splunk " + self.request['systemAuth']} url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote( splunk_vc_name) + "?output_mode=json" logger.debug( "Now running query against url=%s to obtain config information" % (url)) res = self.runHttpRequest( url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name), sslVerify=False) if not res: return #Look under the entry/content section for the relevant information we require, mainly destURL, useLocalAuth and potentially destUsername/destPassword json_dict = json.loads(res.text)['entry'][0]['content'] #self.response.write(str(json_dict) + "\n\n\n") useLocalAuth = False if 'useLocalAuth' in json_dict: if json_dict['useLocalAuth'].lower( ) == 't' or json_dict['useLocalAuth'].lower() == "true": useLocalAuth = True if not useLocalAuth: if not 'destUsername' in json_dict or not 'destPassword' in json_dict or not 'destURL' in json_dict: logger.error( "Missing one of destUsername, destPassword or destURL from the splunk version control restore stanza, and useLocalAuth is not true, invalid configuration" ) self.response.write( "Missing one of destUsername, destPassword or destURL from the splunk version control restore stanza, and useLocalAuth is not true, invalid configuration" ) return destUsername = json_dict['destUsername'] destPassword = json_dict['destPassword'] if destPassword.find("password:"******"Missing one of destURL from the splunk version control restore stanza, invalid configuration" ) self.response.write( "Missing destURL from the splunk version control restore stanza, invalid configuration" ) return destURL = json_dict['destURL'] sslVerify = False if 'sslVerify' in json_dict: sslVerify = json_dict['sslVerify'] headers = {} auth = None if useLocalAuth: headers = {"Authorization": "Splunk " + self.request['systemAuth']} else: auth = HTTPBasicAuth(destUsername, destPassword) if 'remoteAppName' in json_dict and json_dict['remoteAppName'] != "": remoteAppName = json_dict['remoteAppName'] else: remoteAppName = "SplunkVersionControl" if 'timewait' in json_dict and json_dict['timewait'] != '': try: time_wait = int(json_dict['timewait']) except ValueError: logger.warn( "Time wait value of %s is invalid, not an integer, defaulting to 600 seconds" % (json_dict['timewait'])) time_wait = 600 else: time_wait = 600 if 'requestingAddress' in payload: requestingAddress = payload['requestingAddress'][0] else: requestingAddress = None username, roles = self.query_back_for_user_and_permissions( requestingAddress, payload['Authorization'][0], sslVerify=sslVerify) logger.info("username=%s roles=%s" % (username, roles)) app = payload['app'][0] type = payload['type'][0] obj_name = payload['obj_name'][0] tag = payload['tag'][0] timeout = payload['timeout'][0] if not 'restoreAsUser' in payload: restoreAsUser = '' else: restoreAsUser = payload['restoreAsUser'][0] scope = payload['scope'][0] logger.debug("Converting timeout of argument %s to integer" % timeout) timeout = int(timeout) #We need a little bit of time to index the _audit event that literally just happened #a 30 second delay is annoying but it appears to work...hardcoding this for now logger.info("Sleeping for %s seconds to wait for audit logs" % (timeout)) time.sleep(timeout) logger.info("Sleep completed") starttime = starttime - 60 - timeout json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime, sslVerify=sslVerify) if 'error' in json_res: self.response.write("An error occurred: %s" % (json_res['error'])) return if len(json_res['results']) == 0: logger.warn( "No matching results for audit query using username=%s, remoteAppName=%s on url=%s with starttime of %s" % (username, remoteAppName, destURL, starttime)) self.response.write( "No matching results for audit query using username=%s, remoteAppName=%s on url=%s with starttime of %s" % (username, remoteAppName, destURL, starttime)) return else: #we are at the point where we checked the remote instance and confirmed the user in question was allowed to request a restore, pass control #to the restore class to attempt the actual restore svc_restore_obj = SplunkVersionControlRestore() resList = [{ 'app': app, 'type': type, 'name': obj_name, 'tag': tag, 'scope': scope, 'time': starttime, 'restoreAsUser': restoreAsUser, 'user': username }] #Name is required as part of the config dictionary, session_key is used if useLocalAuth is true in the config json_dict[ 'name'] = "splunkversioncontrol_restore://" + splunk_vc_name json_dict['session_key'] = self.request['systemAuth'] #Check current time and see if anyone is running a restore headers = {"Authorization": "Splunk " + self.request['systemAuth']} curtime = calendar.timegm(time.gmtime()) url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status" res = self.runHttpRequest( url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=False) if not res: return res = json.loads(res.text) #An empty list is good in this case, we are safe to run, if not we have checks to do if not len(res) == 0: if not 'start_time' in res[0]: logger.warn( "Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\"" % (url, payload)) self.runHttpRequest( url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=False) else: kvstore_start_time = res[0]['start_time'] target_time = curtime - time_wait if kvstore_start_time < target_time: logger.warn( "Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)" % (kvstore_start_time, curtime, time_wait, target_time)) #More than 10 minutes ago, delete the entry and move on self.runHttpRequest( url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=False) else: removal_target = kvstore_start_time + time_wait + 1 logger.warn( "Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)" % (kvstore_start_time, curtime, time_wait, removal_target)) self.response.write( "Attempted to run but found a running restore instance with time %s and current time is %s, will delete and move on after current time minus %s seconds (which would be %s) " % (kvstore_start_time, curtime, time_wait, removal_target)) self.response.write( "Please try your restore request again in a minute..." ) return payload = json.dumps({'start_time': curtime}) headers['Content-Type'] = 'application/json' #update kvstore with runtime res = self.runHttpRequest( url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=False) if not res: return res (result, message) = svc_restore_obj.run_script(resList, json_dict) if result == True: self.response.write( "Restore has completed successfully in app %s, object of type %s, with name %s was restored from tag %s, scope %s with restoreAsUser %s and your username of %s" % (app, type, obj_name, tag, scope, restoreAsUser, username)) logger.info( "Restore has completed successfully in app=%s, object of type=%s, with name=%s was restored from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username)) else: self.response.write( "Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) logger.warn( "Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) self.runHttpRequest( url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=False)
def validate_arguments(): val_data = get_validation_data() if 'debugMode' in val_data: debugMode = val_data['debugMode'].lower() if debugMode == "true" or debugMode == "t" or debugMode == "1": logging.getLogger().setLevel(logging.DEBUG) session_key = val_data['session_key'] useLocalAuth = False if 'useLocalAuth' in val_data: useLocalAuth = val_data['useLocalAuth'].lower() if useLocalAuth == "true" or useLocalAuth == "t" or useLocalAuth == "1": useLocalAuth = True logger.debug("useLocalAuth enabled") if val_data['srcURL'] != "https://*****:*****@") logger.debug( "Attempting to replace proxy=%s by subsituting=%s with a password" % (proxies['https'], proxies['https'][start:end])) temp_password = get_password(proxies['https'][start:end], session_key, logger) proxies['https'] = proxies['https'][ 0:start - 9] + temp_password + proxies['https'][end:] try: logger.debug( "Running query against URL %s with username %s proxies_length=%s sslVerify=%s" % (url, srcUsername, len(proxies), sslVerify)) res = requests.get(url, auth=(srcUsername, srcPassword), verify=sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, srcUsername)) if (res.status_code != requests.codes.ok): print_error( "Attempt to validate access to Splunk failed with code %s, reason %s, text %s, on URL %s" % (res.status_code, res.reason, res.text, url)) sys.exit(4) except requests.exceptions.RequestException as e: print_error( "Attempt to validate access to Splunk failed with error %s" % (e)) sys.exit(5) gitRepoURL = val_data['gitRepoURL'] proxy_command = "" git_password = False if gitRepoURL.find("http") == 0: gitRepoHTTP = True if gitRepoURL.find("password:"******"password:"******"@") logger.debug( "Attempting to replace gitRepoURL=%s by subsituting=%s with a password" % (gitRepoURL, gitRepoURL[start:end])) git_password = get_password(gitRepoURL[start:end], session_key, logger) gitRepoURL = gitRepoURL[0:start - 9] + git_password + gitRepoURL[end:] else: gitRepoHTTP = False git_proxies = {} if 'git_proxy' in val_data: git_proxies["https"] = val_data['git_proxy'] if git_proxies['https'].find("password:"******"password:"******"@") logger.debug( "Attempting to replace git_proxy=%s by subsituting=%s with a password" % (git_proxies['https'], git_proxies['https'][start:end])) temp_password = get_password(git_proxies['https'][start:end], session_key, logger) git_proxies['https'] = git_proxies['https'][ 0:start - 9] + temp_password + git_proxies['https'][end:] if gitRepoHTTP and len(git_proxies) > 0: logger.debug( "Adding environment variable HTTPS_PROXY before git commands") proxy_command = "HTTPS_PROXY=" + git_proxies["https"] if platform.system() == "Windows": proxy_command = "set " + proxy_command + " & " else: proxy_command = "export " + proxy_command + " ; " show_passwords = False if 'show_passwords' in val_data: if val_data['show_passwords'].lower( ) == 'true' or val_data['show_passwords'] == "1": show_passwords = True logger.debug('show_passwords is now true due to show_passwords: ' + val_data['show_passwords']) (stdout, stderr, res) = runOSProcess( "%s %s ls-remote %s" % (proxy_command, git_command, gitRepoURL), logger, shell=True) #If we didn't manage to ls-remote perhaps we just need to trust the fingerprint / this is the first run? if res == False and not gitRepoHTTP: if not show_passwords and git_password: stdout = stdout.replace(git_password, "password_removed") stderr = stderr.replace(git_password, "password_removed") logger.error("Possible first run trying again" % (stdout, stderr)) (stdout, stderrout, res) = runOSProcess( ssh_command + " -n -o \"BatchMode yes\" -o StrictHostKeyChecking=no " + gitRepoURL[:gitRepoURL.find(":")], logger) (stdout, stderr, res) = runOSProcess("%s ls-remote %s" % (git_command, gitRepoURL), logger) if res == False: if not show_passwords and git_password: stdout = stdout.replace(git_password, "password_removed") stderr = stderr.replace(git_password, "password_removed") print_error( "Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr)) logger.error( "Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr)) sys.exit(6)