def __init__(self):
'''initialize the data assembler tools'''
self.chapi = CloudHealth()
self.librato = Librato()
self.lock = threading.RLock()
th = threading.Thread(target=self.fill_loop)
th.daemon = True
th.start()
class DataAssembler():
def __init__(self):
'''initialize the data assembler tools'''
self.chapi = CloudHealth()
self.librato = Librato()
self.lock = threading.RLock()
th = threading.Thread(target=self.fill_loop)
th.daemon = True
th.start()
# Consume tag compliance data from jenkins
def get_tag_compliance_data(self):
'''get tag compliance data from jenkins'''
request = urllib2.Request("http://jenkins.csi.dgs.io/job/AWS%20Rules%20Invocation/lastSuccessfulBuild/artifact/awsTagCompliance.json")
response = urllib2.urlopen(request)
page = response.read()
j = json.loads(page)
data["AWSTagCompliance"] = j
# Consume data from librato API
def fill_collectd_data(self):
''' get collectd data from librato'''
sourcecount = len(self.librato.get_librato_data('s("collectd.cpu.0.cpu.system","*")')["measurements"])
data["TurbineCollectd"] = {"Total": sourcecount}
# Consume data from CH
def _ensure_cloud_health_data(self):
'''check that CloudHealth data is present'''
if 'CloudHealth' in data:
return
with self.lock:
data['CloudHealth'] = self.fill_cloud_health_data()
def get_report_list(self):
'''use chapi to get a list of the reports'''
report = self.chapi.get_list_of_custom_reports()
return report
def get_report_data(self, url):
'''get the data of a specific report'''
print "GET REPORT: " + url
report = self.chapi.get_cloudhealth_data(url)
return report
def fill_cloud_health_data(self):
'''fill a data structure with data from CloudHealth'''
ch_data = {}
def perform(reportKey):
'''get the report as normalized data'''
# get a report from CloudHealth
report = self.chapi.get_normalized_report(reportKey["url"])
name = None
# name formatting
if reportKey["friendlyName"] == "":
name = report.keys()[0]
ch_data[name] = report[name]
else:
name = reportKey["friendlyName"]
nameReport = self.chapi.get_normalized_report(reportKey["url"])
#pprint.pprint(nameReport)
ch_data[name] = report[nameReport.keys()[0]]
#pprint.pprint(_cloudhealth_reports)
with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
map(perform, _cloudhealth_reports)
#pprint.pprint(ch_data)
return ch_data
def get_raw_data(self):
'''get the raw data of all reports'''
self._ensure_cloud_health_data()
self.get_tag_compliance_data()
self.fill_collectd_data()
return data;
def fill_loop(self):
'''fill the data object with cloudheatlt data'''
while True:
with self.lock:
data['CloudHealth'] = self.fill_cloud_health_data()
time.sleep(1000)
def get_all_data(self):
'''get data from all sources'''
self._ensure_cloud_health_data()
self.get_tag_compliance_data()
self.fill_collectd_data()
return data
