def __init__(self, username, password, title, space, content, parent_page):
self.wiki_url = "https://docs.engineering.redhat.com"
self.username = username
self.password = password
self.api = Api(self.wiki_url, self.username, self.password)
self.page_name = title
self.general_content = content
self.space = space
self.parent_page = parent_page
self.content = ""
self.page_all_info = {}
def connect(self):
url = self.get_config_or_ask('url', label='URL (usually ends in "/wiki")')
username = self.get_config_or_ask('username', label='Username (usually without the @domain part)')
password = self.get_config_or_ask('password', label=f"Password for {username} (won't be displayed)", hidden=True)
# password = self.ask(f"Password for {username} (won't be displayed)", hidden=True)
with self.info('Conecting...'):
try:
self.api = Api(url, username, password)
except Exception as ex:
the_type = type(ex)
self.info(f'{the_type}: {ex}')
return
else:
self.persist_unsaved_config()
class ConfluenceClient():
def __init__(self, confluence_username, confluence_password, title, space,
content, parent_page):
self.wiki_url = "https://docs.engineering.redhat.com"
self.confluence_username = confluence_username
self.confluence_password = confluence_password
self.api = Api(self.wiki_url, self.confluence_username,
self.confluence_password)
self.page_name = title
self.general_content = content
self.space = space
self.parent_page = parent_page
self.content = ""
self.page_all_info = {}
def create_update_page(self):
self.get_page_content()
if self.content.find("it does not exist.") > 0 or self.content.find(
"QE") < 0:
print "==== Will add page ==="
print "==== The page title: ", self.page_name
self.api.addpage(self.page_name,
self.space,
self.general_content,
parentpage=self.parent_page)
else:
self.get_page_all()
print "==== Will update page ==="
print "==== The page title: ", self.page_name
update_confluence_client = confluence_rest_api_client.ConfluenceClientForUpdatePage(
self.space, self.page_name, self.wiki_url,
self.confluence_username, self.confluence_password)
update_confluence_client.update_page(self.page_all_info,
self.general_content)
def get_page_content(self):
try:
self.content = self.api.getpagecontent(self.page_name, self.space)
except Exception, error:
self.content = str(error)
def main():
global IS_CW
global JIRA_ID
if APP_IDS != None:
for APP_ID in APP_IDS.split(','):
APP_V = get_latest_version(APP_ID)
if(APP_V == "###"):
print "Any version of the app",APP_ID,"is not available on 1sot"
else:
print "For app:",APP_ID
print "latest version on 1sot is:",APP_V
if query_yes_no("Change the version?","yes"):
sys.stdout.write("Enter the version: ")
choice = raw_input()
APP_V = choice
APP_VS[APP_ID] = APP_V
print ""
APP_IDSS.append(APP_ID)
print "*+Pre-checks for stack+*"
print " - stack: ",STACK
content = ""
try:
urllib3.contrib.pyopenssl.inject_into_urllib3()
urllib3.disable_warnings()
user_agent = {'user-agent': 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) ..'}
http = urllib3.PoolManager(10, headers=user_agent)
url = 'https://api.github.com/repos/SplunkStorm/stax/contents/'+STACK+'.json?access_token='+TOKEN
result = http.request("GET",url)
if result.status == 200:
req = json.loads(result.data)
content = base64.decodestring(req['content'])
j = json.loads(content)
print " - Splunk Account: ",j['attributes']['cloud_vendor']['account']
print " - Splunk Version: ",j['attributes']['splunkwhisper']['splunk_version']
print " - Is the stack CloudWorks: *No*"
IS_CW = 0
elif result.status == 404:
adhoc_sh = dns.resolver.query(STACK+'.splunkcloud.com', 'CNAME')[0].target[0]
sh_id = dns.resolver.query(adhoc_sh+'.'+STACK+'.splunkcloud.com', 'CNAME')[0].target.__str__()
if "sh" in sh_id:
IS_CW = 1
print " - Is the stack CloudWorks: *Yes*"
else:
print "*Stack is not available*"
else:
print "Github Error with response code :*"+result.status.__str__()+"*"
except:
print "Github Error: *Run the script with sudo* or invalid stack name or check the token value in variables.py"
if PASSWORD != "###":
sf,rf = check_SF_RF()
print " - Search Factor: ",sf
print " - Replication Factor: ",rf
if sf == "False" or rf == "False":
ERROR.append("SF/RF")
stack_available=1
try:
answers = dns.resolver.query(STACK+'.splunkcloud.com', 'CNAME')
if "shc" not in answers[0].target[0]:
print " - adhoc sh: ", answers[0].target
SHs['adhoc'] = answers[0].target[0]
else:
print "*"+STACK+'.splunkcloud.com '+"is pointing to SHC*"
except:
print "*"+STACK+'.splunkcloud.com '+"DNS is not available*"
ERROR.append("STACK")
stack_available=0
if stack_available==1:
try:
answers = dns.resolver.query('es-'+STACK+'.splunkcloud.com', 'CNAME')
print " - es sh: ", answers[0].target
SHs['es'] = answers[0].target[0]
except:
print " - es sh: *DNS not available*"
try:
answers = dns.resolver.query('itsi-'+STACK+'.splunkcloud.com', 'CNAME')
print " - itsi sh: ", answers[0].target
SHs['itsi'] = answers[0].target[0]
except:
print " - itsi sh: *DNS not available*"
try:
answers = dns.resolver.query('vmware-'+STACK+'.splunkcloud.com', 'CNAME')
print " - vmware sh:", answers[0].target
SHs['vmware'] = answers[0].target[0]
except:
print " - vmware sh: *DNS not available*"
try:
answers = dns.resolver.query('pci-'+STACK+'.splunkcloud.com', 'CNAME')
print " - pci sh:", answers[0].target
SHs['pci'] = answers[0].target[0]
except:
print " - pci sh: *DNS not available*"
try:
answers = dns.resolver.query('exchange-'+STACK+'.splunkcloud.com', 'CNAME')
print " - exchange sh:", answers[0].target
SHs['exchange'] = answers[0].target[0]
except:
print " - exchange sh: *DNS not available*"
print ""
if APP_IDS != None:
appcert_flg = 0
checked_app_ids = []
for APP_ID in APP_IDSS:
APP_V = APP_VS[APP_ID]
print "*+Pre-checks for "+APP_ID+" v"+APP_V+"+*"
tmp = check_on_1sot(APP_ID,APP_V)
if tmp != "not":
print " - 1sot: ",APP_ID+"_"+APP_V+tmp
else:
print " - The app "+APP_ID+" v"+APP_V+" is not available on 1sot"
ERROR.append("1SOT")
appcert_f = 0
options = {'server': jira_server, 'verify':False}
jira = JIRA(options=options, basic_auth=(jira_user, jira_password))
query = 'project = APPCERT AND status = Closed AND resolution = "Fixed" AND text ~ "'+APP_ID+' v'+APP_V+'"'
issues = jira.search_issues(query)
if len(issues) != 0:
for issue in issues:
print " - APPCERT: ",issue.__str__()+", status: "+issue.fields.status.__str__()+", resolution: ", issue.fields.resolution.__str__()
appcert_f = 1
if appcert_f == 0:
print "*Automation failed to find APPCERT JIRA*"
ERROR.append("APPCERT")
appcert_flg = 1
splunkbase = check_on_splunkbase(APP_ID,APP_V)
if splunkbase != "ERROR_404":
print " - Splunk-Base: available"
print " - Supported Splunk versions: ",splunkbase
else:
print " - The app "+APP_ID+" v"+APP_V+" is not available on Splunk-Base"
folder_name, _status = self_service_check(APP_ID)
if "ERROR" not in _status:
if _status == "appmgmt_phase":
print " - *The app supports self-service installation*"
elif _status == "assisted":
print " - self-service installation: No"
elif _status == "unknown":
print " - *This app is not yet vetted to install on Splunk Cloud*"
elif _status == "rejected":
print " - *This app is not yet vetted to install on Splunk Cloud*"
if PASSWORD != "###":
#folder_name = get_app_folder_name(APP_ID+"_"+APP_V+tmp)
if "ERROR" not in folder_name:
print " - App directory name: ",folder_name
for key, value in SHs.iteritems():
installed,restart_req,current_ver = get_install_status(folder_name,value)
if installed == "yes":
print " - *The app "+APP_ID+" is already installed on "+key+" SH with "+current_ver+" version.*"
print " - need Splunk restart after upgrade: "+restart_req
else:
print " - Is it already installed on "+key+" SH: No"
url = CONFLUENCE_URL
api = Api(url, jira_user, jira_password)
text = api.getpagecontent("Splunk Cloud Apps Information List","CLOUDOPS")
soup = BeautifulSoup(text.encode("utf-8"),"html.parser")
ids=""
if len(soup.findAll("span"))!=0:
for span_tag in soup.findAll("span"):
if span_tag!= None and span_tag.find(text=True) != None:
tmp = span_tag.find(text=True)
span_tag.replace_with(tmp)
if len(soup.findAll("p"))!=0:
for span_tag in soup.findAll("p"):
if span_tag!= None and span_tag.find(text=True) != None:
tmp = span_tag.find(text=True)
span_tag.replace_with(tmp)
if len(soup.findAll("br"))!=0:
for span_tag in soup.findAll("br"):
if span_tag!= None and span_tag.find(text=True) != None:
tmp = span_tag.find(text=True)
span_tag.replace_with(tmp)
if len(soup.findAll("td",text=APP_ID))!=0:
for nodes in soup.findAll("td",text=APP_ID):
allnodes = nodes.parent.findAll(recursive=False)
if allnodes[0].find(text=True) == APP_ID:
print " - APP DETAILS"
print "\tApp-ID: ",allnodes[0].find(text=True).replace(" ", "")
sys.stdout.write("\tcan be installed on: ")
if allnodes[4].find(text=True) != None and "true" in allnodes[4].find(text=True).replace(" ", "").lower():
sys.stdout.write("sh ")
if allnodes[5].find(text=True) != None and "true" in allnodes[5].find(text=True).replace(" ", "").lower():
sys.stdout.write("c0m1 ")
if allnodes[6].find(text=True) != None and "true" in allnodes[6].find(text=True).replace(" ", "").lower():
sys.stdout.write("hfw ")
if allnodes[7].find(text=True) != None and "true" in allnodes[7].find(text=True).replace(" ", "").lower():
sys.stdout.write("ufw ")
print ""
if allnodes[12].find(text=True) != None and allnodes[12].find(text=True).replace(" ", "").strip().replace(" ","") != "":
print "\tdependent apps: ",allnodes[12].find(text=True).replace(" ", "")
if allnodes[12].find(text=True) != None:
ids = allnodes[12].find(text=True).replace(" ", "")
else:
print "*App is not available on confluence page*"
ids = ids.split('|')
for _id in ids:
_id = ''.join(c for c in _id if c.isdigit())
if len(soup.findAll("td",text=_id))!=0:
for nodes in soup.findAll("td",text=_id):
allnodes = nodes.parent.findAll(recursive=False)
if allnodes[0].find(text=True) == _id:
print " - APP DETAILS for dependent app ",_id
print "\tApp-ID: ",allnodes[0].find(text=True).replace(" ", "")
sys.stdout.write("\tcan be installed on: ")
if allnodes[4].find(text=True) != None and "true" in allnodes[4].find(text=True).replace(" ", "").lower():
sys.stdout.write("sh ")
if allnodes[5].find(text=True) != None and "true" in allnodes[5].find(text=True).replace(" ", "").lower():
sys.stdout.write("c0m1 ")
if allnodes[6].find(text=True) != None and "true" in allnodes[6].find(text=True).replace(" ", "").lower():
sys.stdout.write("hfw ")
if allnodes[7].find(text=True) != None and "true" in allnodes[7].find(text=True).replace(" ", "").lower():
sys.stdout.write("ufw ")
print ""
if allnodes[12].find(text=True) != None and allnodes[12].find(text=True).replace(" ", "").strip().replace(" ","") != "":
print "\tdependent apps: ",allnodes[12].find(text=True).replace(" ", "")
_v = get_latest_version(_id)
print "\tlatest version: "+_v
splunkbase = check_on_splunkbase(_id,_v)
if splunkbase != "ERROR_404":
print "\tSplunk-Base: available"
print "\tSupported Splunk versions: ",splunkbase
else:
print "\tThe app "+_id+" v"+_v+" is not available on Splunk-Base"
tmp = check_on_1sot(_id,_v)
if tmp != "not":
print "\t1sot: ",_id+"_"+_v+tmp
else:
print "\tThe app "+_id+" v"+_v+" is not available on 1sot"
if PASSWORD != "###":
folder_name, _status = self_service_check(_id)
if "ERROR" not in _status:
if _status == "appmgmt_phase":
print "\t*The app supports self-service installation*"
elif _status == "assisted":
print "\tself-service installation: No"
elif _status == "unknown":
print "\t*This app is not yet vetted to install on Splunk Cloud*"
elif _status == "rejected":
print "\t*This app is not yet vetted to install on Splunk Cloud*"
if "ERROR" not in folder_name:
print "\tApp directory name: ",folder_name
for key, value in SHs.iteritems():
installed,restart_req,current_ver = get_install_status(folder_name,value)
if installed == "yes":
print "\t*The app "+_id+" is already installed on "+key+" SH with "+current_ver+" version.*"
print "\tneed Splunk restart after upgrade: "+restart_req
else:
print "\tIs it already installed on "+key+" SH: No"
appcert_f = 0
options = {'server': jira_server}
jira = JIRA(options=options, basic_auth=(jira_user, jira_password))
query = 'project = APPCERT AND status = Closed AND resolution = "Fixed" AND text ~ "'+_id+' v'+_v+'"'
issues = jira.search_issues(query)
if len(issues) != 0:
for issue in issues:
print "\tAPPCERT: ",issue.__str__()+", status: "+issue.fields.status.__str__()+", resolution: ", issue.fields.resolution.__str__()
appcert_f = 1
if appcert_f == 0:
print "\t*Automation failed to find APPCERT JIRA for dependent app",_id+"*"
else:
print "*dependent app is not available on confluence page*"
print ""
if appcert_flg == 1:
print "Gone through below JIRA to get the remaining APPCERT JIRA"
sys.stdout.write("Enter the app install JIRA ID (CO-12345): ")
JIRA_ID = raw_input()
issue = jira.issue(JIRA_ID)
for link in issue.fields.issuelinks:
#print link.key
if hasattr(link, "outwardIssue"):
outwardIssue = link.outwardIssue
print "\t",outwardIssue.key+" "+outwardIssue.fields.summary.__str__()
print "\tstatus: ",outwardIssue.fields.status,"\tresolution: ",jira.issue(outwardIssue.key).fields.resolution.__str__()
print ""
if hasattr(link, "inwardIssue"):
inwardIssue = link.inwardIssue
print "\t",inwardIssue.key+" "+inwardIssue.fields.summary.__str__()
print "\tstatus: ",inwardIssue.fields.status,"\tresolution: ",jira.issue(inwardIssue.key).fields.resolution.__str__()
print ""
class App(AppBase):
title = 'Confluence'
def init(self):
self.current_page = None
self.current_space = None
self.spaces = {}
self.pages_tree = {}
self.pages = {}
self.entries = []
self.connect()
self.load_spaces()
self.enqueue(self.cmd__l)
def connect(self):
url = self.get_config_or_ask('url', label='URL (usually ends in "/wiki")')
username = self.get_config_or_ask('username', label='Username (usually without the @domain part)')
password = self.get_config_or_ask('password', label=f"Password for {username} (won't be displayed)", hidden=True)
# password = self.ask(f"Password for {username} (won't be displayed)", hidden=True)
with self.info('Conecting...'):
try:
self.api = Api(url, username, password)
except Exception as ex:
the_type = type(ex)
self.info(f'{the_type}: {ex}')
return
else:
self.persist_unsaved_config()
def load_spaces(self):
with self.info('Loading spaces list'):
for entry in self.api.listspaces():
name = entry['name']
key = entry['key']
status = entry['status']
if status == 'CURRENT':
try:
pages_tree = self.load_pages(key)
except Exception as ex:
print(f' {ex}')
del self.pages_tree[key]
continue
self.spaces[key] = {
'key': key,
'name': name,
'type': entry['type'],
'url': entry['url'],
'pages': pages_tree,
}
# TODO: persist it somewhere? It doesn't change that much.
def load_pages(self, space_key):
tree = self.pages_tree[space_key] = defaultdict(list)
for page, _ in self.api.listpages(space_key):
parent_id = page['parentId']
tree[parent_id].append(page)
self.pages[page['id']] = page
return tree
def cmd__l(self):
monitor = self.open_monitor('Spaces')
for space in self.spaces.values():
num_pages = len(space["pages"])
line = f'{space["key"]:>10}: {space["name"]:>30} ({num_pages} pages)'
monitor.write(line)
def cmd__v(self, key, debug=False):
"""
View page.
Usage: v <space_key> <page title>
For now, you must write down the entire
page title. Case sensitive. This surely
will change in the future.
"""
debug = bool(debug)
if key in self.spaces:
space = self.spaces[key]
self.current_space = key
pages_tree = space['pages']
page = pages_tree['0'][0]
else:
space = self.spaces[self.current_space]
index = int(key)
pages_tree = space['pages']
page = self.entries[index]
page_id = page['id']
children = pages_tree.get(page_id, None)
monitor = self.open_monitor(page['title'])
self.entries = []
parent_id = page['parentId']
index = 0
if parent_id != '0':
parent = self.pages[parent_id]
self.entries.append(parent)
monitor.write(f'{index:>4}: {parent["title"]} (parent)')
monitor.hr()
index += 1
if children:
monitor.h2('Children pages')
for index, child in enumerate(children, index):
monitor.write(f'{index:>4}: {child["title"]}')
self.entries.append(child)
else:
monitor.write('No children pages.')
monitor.hr()
self.enqueue(self.load_page, page, monitor, debug)
def cmd__b(self):
"""
Go back one page.
Same as "v 0" on most cases.
"""
if len(self.entries) > 0:
return self.cmd__v(0)
else:
self.info('No entry "0" to go')
def load_page(self, page, monitor, debug=False):
content_page = self.api.getpage(page['title'], self.current_space)
content = content_page['content']
monitor.h1(page['title'])
parser = Parser(monitor)
parser.debug = debug
parser.feed(content)