def validate_arguments():
val_data = get_validation_data()
if 'debugMode' in val_data:
debugMode = val_data['debugMode'].lower()
if debugMode == "true" or debugMode == "t":
logging.getLogger().setLevel(logging.DEBUG)
useLocalAuth = False
if 'useLocalAuth' in val_data:
useLocalAuth = val_data['useLocalAuth'].lower()
if useLocalAuth == "true" or useLocalAuth == "t":
useLocalAuth = True
logger.debug("useLocalAuth enabled")
if val_data['destURL'] != "https://*****:*****@")
logger.debug(
"Attempting to replace proxy=%s by subsituting=%s with a password"
% (proxies['https'], proxies['https'][start:end]))
temp_password = get_password(proxies['https'][start:end],
session_key, logger)
proxies['https'] = proxies['https'][
0:start - 9] + temp_password + proxies['https'][end:]
try:
logger.debug(
"Running query against URL %s with username %s proxies_length=%s"
% (url, destUsername, len(proxies), sslVerify))
res = requests.get(url,
auth=(destUsername, destPassword),
verify=sslVerify,
proxies=proxies)
logger.debug("End query against URL %s with username %s" %
(url, destUsername))
if (res.status_code != requests.codes.ok):
print_error(
"Attempt to validate access to Splunk failed with code %s, reason %s, text %s on URL %s"
% (res.status_code, res.reason, res.text, url))
sys.exit(1)
except requests.exceptions.RequestException as e:
print_error(
"Attempt to validate access to Splunk failed with error %s" %
(e))
sys.exit(1)
gitRepoURL = val_data['gitRepoURL']
if 'git_command' in val_data:
git_command = val_data['git_command'].strip()
logger.debug("Overriding git command to %s" % (git_command))
else:
git_command = "git"
if 'ssh_command' in val_data:
ssh_command = val_data['ssh_command'].strip()
logger.debug("Overriding ssh command to %s" % (ssh_command))
else:
ssh_command = "ssh"
(stdout, stderr,
res) = runOSProcess("%s ls-remote %s" % (git_command, gitRepoURL), logger)
#If we didn't manage to ls-remote perhaps we just need to trust the fingerprint / this is the first run?
if res == False:
(stdout, stderrout, res) = runOSProcess(
ssh_command +
" -n -o \"BatchMode yes\" -o StrictHostKeyChecking=no " +
gitRepoURL[:gitRepoURL.find(":")], logger)
(stdout, stderr, res) = runOSProcess("%s ls-remote %s" % (git_command),
logger)
if res == False:
print_error(
"Failed to validate the git repo URL, stdout of '%s', stderr of '%s'"
% (stdout, stderr))
sys.exit(1)
def handle_POST(self):
starttime = calendar.timegm(time.gmtime())
payload = six.moves.urllib.parse.parse_qs(self.request['payload'])
#self.response.write(str(payload) + "\n")
#currently we only receive the Splunk authorization key, so obtain that
req_attributes = [
'Authorization', 'splunk_vc_name', 'app', 'type', 'obj_name',
'tag', 'scope', 'timeout'
]
for attr in req_attributes:
if attr not in payload:
#Don't log the authorization key if it was sent
if 'Authorization' in payload:
del payload['Authorization']
logger.error(
"Received remote call but attr=%s was missing from arguments, received=\"%s\""
% (attr, payload))
self.response.write(
"Received remote call but attr=%s was missing from arguments, received=\"%s\""
% (attr, payload))
return
splunk_vc_name = payload['splunk_vc_name'][0]
#self.response.write(splunk_vc_name + "\n\n\n")
#Now run queries locally to check if the mentioned config matches an existing backup name
headers = {"Authorization": "Splunk " + self.request['systemAuth']}
url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote(
splunk_vc_name) + "?output_mode=json"
logger.debug(
"Now running query against url=%s to obtain config information" %
(url))
res = self.runHttpRequest(
url,
headers,
None,
"get",
"querying the inputs for splunkversioncontrol_restore with name %s"
% (splunk_vc_name),
sslVerify=False)
if not res:
return
#Look under the entry/content section for the relevant information we require, mainly destURL, useLocalAuth and potentially destUsername/destPassword
json_dict = json.loads(res.text)['entry'][0]['content']
#self.response.write(str(json_dict) + "\n\n\n")
useLocalAuth = False
if 'useLocalAuth' in json_dict:
if json_dict['useLocalAuth'].lower(
) == 't' or json_dict['useLocalAuth'].lower() == "true":
useLocalAuth = True
if not useLocalAuth:
if not 'destUsername' in json_dict or not 'destPassword' in json_dict or not 'destURL' in json_dict:
logger.error(
"Missing one of destUsername, destPassword or destURL from the splunk version control restore stanza, and useLocalAuth is not true, invalid configuration"
)
self.response.write(
"Missing one of destUsername, destPassword or destURL from the splunk version control restore stanza, and useLocalAuth is not true, invalid configuration"
)
return
destUsername = json_dict['destUsername']
destPassword = json_dict['destPassword']
if destPassword.find("password:"******"Missing one of destURL from the splunk version control restore stanza, invalid configuration"
)
self.response.write(
"Missing destURL from the splunk version control restore stanza, invalid configuration"
)
return
destURL = json_dict['destURL']
sslVerify = False
if 'sslVerify' in json_dict:
sslVerify = json_dict['sslVerify']
headers = {}
auth = None
if useLocalAuth:
headers = {"Authorization": "Splunk " + self.request['systemAuth']}
else:
auth = HTTPBasicAuth(destUsername, destPassword)
if 'remoteAppName' in json_dict and json_dict['remoteAppName'] != "":
remoteAppName = json_dict['remoteAppName']
else:
remoteAppName = "SplunkVersionControl"
if 'timewait' in json_dict and json_dict['timewait'] != '':
try:
time_wait = int(json_dict['timewait'])
except ValueError:
logger.warn(
"Time wait value of %s is invalid, not an integer, defaulting to 600 seconds"
% (json_dict['timewait']))
time_wait = 600
else:
time_wait = 600
if 'requestingAddress' in payload:
requestingAddress = payload['requestingAddress'][0]
else:
requestingAddress = None
username, roles = self.query_back_for_user_and_permissions(
requestingAddress,
payload['Authorization'][0],
sslVerify=sslVerify)
logger.info("username=%s roles=%s" % (username, roles))
app = payload['app'][0]
type = payload['type'][0]
obj_name = payload['obj_name'][0]
tag = payload['tag'][0]
timeout = payload['timeout'][0]
if not 'restoreAsUser' in payload:
restoreAsUser = ''
else:
restoreAsUser = payload['restoreAsUser'][0]
scope = payload['scope'][0]
logger.debug("Converting timeout of argument %s to integer" % timeout)
timeout = int(timeout)
#We need a little bit of time to index the _audit event that literally just happened
#a 30 second delay is annoying but it appears to work...hardcoding this for now
logger.info("Sleeping for %s seconds to wait for audit logs" %
(timeout))
time.sleep(timeout)
logger.info("Sleep completed")
starttime = starttime - 60 - timeout
json_res = self.runSearchJob(destURL,
remoteAppName,
headers,
auth,
username,
starttime,
sslVerify=sslVerify)
if 'error' in json_res:
self.response.write("An error occurred: %s" % (json_res['error']))
return
if len(json_res['results']) == 0:
logger.warn(
"No matching results for audit query using username=%s, remoteAppName=%s on url=%s with starttime of %s"
% (username, remoteAppName, destURL, starttime))
self.response.write(
"No matching results for audit query using username=%s, remoteAppName=%s on url=%s with starttime of %s"
% (username, remoteAppName, destURL, starttime))
return
else:
#we are at the point where we checked the remote instance and confirmed the user in question was allowed to request a restore, pass control
#to the restore class to attempt the actual restore
svc_restore_obj = SplunkVersionControlRestore()
resList = [{
'app': app,
'type': type,
'name': obj_name,
'tag': tag,
'scope': scope,
'time': starttime,
'restoreAsUser': restoreAsUser,
'user': username
}]
#Name is required as part of the config dictionary, session_key is used if useLocalAuth is true in the config
json_dict[
'name'] = "splunkversioncontrol_restore://" + splunk_vc_name
json_dict['session_key'] = self.request['systemAuth']
#Check current time and see if anyone is running a restore
headers = {"Authorization": "Splunk " + self.request['systemAuth']}
curtime = calendar.timegm(time.gmtime())
url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status"
res = self.runHttpRequest(
url,
headers,
None,
"get",
"checking kvstore collection splunkversioncontrol_rest_restore_status",
sslVerify=False)
if not res:
return
res = json.loads(res.text)
#An empty list is good in this case, we are safe to run, if not we have checks to do
if not len(res) == 0:
if not 'start_time' in res[0]:
logger.warn(
"Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\""
% (url, payload))
self.runHttpRequest(
url,
headers,
None,
'delete',
'wiping kvstore splunkversioncontrol_rest_restore_status',
sslVerify=False)
else:
kvstore_start_time = res[0]['start_time']
target_time = curtime - time_wait
if kvstore_start_time < target_time:
logger.warn(
"Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)"
% (kvstore_start_time, curtime, time_wait,
target_time))
#More than 10 minutes ago, delete the entry and move on
self.runHttpRequest(
url,
headers,
None,
"delete",
"wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period"
% (kvstore_start_time, target_time),
sslVerify=False)
else:
removal_target = kvstore_start_time + time_wait + 1
logger.warn(
"Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)"
% (kvstore_start_time, curtime, time_wait,
removal_target))
self.response.write(
"Attempted to run but found a running restore instance with time %s and current time is %s, will delete and move on after current time minus %s seconds (which would be %s) "
% (kvstore_start_time, curtime, time_wait,
removal_target))
self.response.write(
"Please try your restore request again in a minute..."
)
return
payload = json.dumps({'start_time': curtime})
headers['Content-Type'] = 'application/json'
#update kvstore with runtime
res = self.runHttpRequest(
url,
headers,
payload,
'post',
'updating kvstore collection splunkversioncontrol_rest_restore_status',
sslVerify=False)
if not res:
return res
(result, message) = svc_restore_obj.run_script(resList, json_dict)
if result == True:
self.response.write(
"Restore has completed successfully in app %s, object of type %s, with name %s was restored from tag %s, scope %s with restoreAsUser %s and your username of %s"
%
(app, type, obj_name, tag, scope, restoreAsUser, username))
logger.info(
"Restore has completed successfully in app=%s, object of type=%s, with name=%s was restored from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s"
%
(app, type, obj_name, tag, scope, restoreAsUser, username))
else:
self.response.write(
"Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s"
% (app, type, obj_name, tag, scope, restoreAsUser,
username, message))
logger.warn(
"Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s"
% (app, type, obj_name, tag, scope, restoreAsUser,
username, message))
self.runHttpRequest(
url,
headers,
None,
'delete',
'wiping kvstore splunkversioncontrol_rest_restore_status after completed run',
sslVerify=False)
def validate_arguments():
val_data = get_validation_data()
if 'debugMode' in val_data:
debugMode = val_data['debugMode'].lower()
if debugMode == "true" or debugMode == "t" or debugMode == "1":
logging.getLogger().setLevel(logging.DEBUG)
session_key = val_data['session_key']
useLocalAuth = False
if 'useLocalAuth' in val_data:
useLocalAuth = val_data['useLocalAuth'].lower()
if useLocalAuth == "true" or useLocalAuth == "t" or useLocalAuth == "1":
useLocalAuth = True
logger.debug("useLocalAuth enabled")
if val_data['srcURL'] != "https://*****:*****@")
logger.debug(
"Attempting to replace proxy=%s by subsituting=%s with a password"
% (proxies['https'], proxies['https'][start:end]))
temp_password = get_password(proxies['https'][start:end],
session_key, logger)
proxies['https'] = proxies['https'][
0:start - 9] + temp_password + proxies['https'][end:]
try:
logger.debug(
"Running query against URL %s with username %s proxies_length=%s sslVerify=%s"
% (url, srcUsername, len(proxies), sslVerify))
res = requests.get(url,
auth=(srcUsername, srcPassword),
verify=sslVerify,
proxies=proxies)
logger.debug("End query against URL %s with username %s" %
(url, srcUsername))
if (res.status_code != requests.codes.ok):
print_error(
"Attempt to validate access to Splunk failed with code %s, reason %s, text %s, on URL %s"
% (res.status_code, res.reason, res.text, url))
sys.exit(4)
except requests.exceptions.RequestException as e:
print_error(
"Attempt to validate access to Splunk failed with error %s" %
(e))
sys.exit(5)
gitRepoURL = val_data['gitRepoURL']
proxy_command = ""
git_password = False
if gitRepoURL.find("http") == 0:
gitRepoHTTP = True
if gitRepoURL.find("password:"******"password:"******"@")
logger.debug(
"Attempting to replace gitRepoURL=%s by subsituting=%s with a password"
% (gitRepoURL, gitRepoURL[start:end]))
git_password = get_password(gitRepoURL[start:end], session_key,
logger)
gitRepoURL = gitRepoURL[0:start -
9] + git_password + gitRepoURL[end:]
else:
gitRepoHTTP = False
git_proxies = {}
if 'git_proxy' in val_data:
git_proxies["https"] = val_data['git_proxy']
if git_proxies['https'].find("password:"******"password:"******"@")
logger.debug(
"Attempting to replace git_proxy=%s by subsituting=%s with a password"
% (git_proxies['https'], git_proxies['https'][start:end]))
temp_password = get_password(git_proxies['https'][start:end],
session_key, logger)
git_proxies['https'] = git_proxies['https'][
0:start - 9] + temp_password + git_proxies['https'][end:]
if gitRepoHTTP and len(git_proxies) > 0:
logger.debug(
"Adding environment variable HTTPS_PROXY before git commands")
proxy_command = "HTTPS_PROXY=" + git_proxies["https"]
if platform.system() == "Windows":
proxy_command = "set " + proxy_command + " & "
else:
proxy_command = "export " + proxy_command + " ; "
show_passwords = False
if 'show_passwords' in val_data:
if val_data['show_passwords'].lower(
) == 'true' or val_data['show_passwords'] == "1":
show_passwords = True
logger.debug('show_passwords is now true due to show_passwords: ' +
val_data['show_passwords'])
(stdout, stderr, res) = runOSProcess(
"%s %s ls-remote %s" % (proxy_command, git_command, gitRepoURL),
logger,
shell=True)
#If we didn't manage to ls-remote perhaps we just need to trust the fingerprint / this is the first run?
if res == False and not gitRepoHTTP:
if not show_passwords and git_password:
stdout = stdout.replace(git_password, "password_removed")
stderr = stderr.replace(git_password, "password_removed")
logger.error("Possible first run trying again" % (stdout, stderr))
(stdout, stderrout, res) = runOSProcess(
ssh_command +
" -n -o \"BatchMode yes\" -o StrictHostKeyChecking=no " +
gitRepoURL[:gitRepoURL.find(":")], logger)
(stdout, stderr,
res) = runOSProcess("%s ls-remote %s" % (git_command, gitRepoURL),
logger)
if res == False:
if not show_passwords and git_password:
stdout = stdout.replace(git_password, "password_removed")
stderr = stderr.replace(git_password, "password_removed")
print_error(
"Failed to validate the git repo URL, stdout of '%s', stderr of '%s'"
% (stdout, stderr))
logger.error(
"Failed to validate the git repo URL, stdout of '%s', stderr of '%s'"
% (stdout, stderr))
sys.exit(6)
