def get(cls, *args, **kwargs):
"""
:param args:
:param kwargs:
:return TxModel:
"""
if args:
kwargs[inspect(cls).primary_key[0].name] = args[0]
item = cls.query(**kwargs).first()
App.db().close()
return item
def get(cls, *args, **kwargs):
"""
:param args:
:param kwargs:
:return TxModel:
"""
if args:
kwargs[inspect(cls).primary_key[0].name] = args[0]
item = cls.query(**kwargs).first()
App.db().close()
return item
def query(cls, **kwargs):
items = App.db().query(cls).filter_by(**kwargs)
return items
def delete(self):
App.db().delete(self)
App.db().commit()
App.db().close()
def update(self):
App.db().merge(self)
App.db().commit()
App.db().close()
def insert(self):
App.db().add(self)
App.db().commit()
App.db().close()
def query(cls, **kwargs):
items = App.db().query(cls).filter_by(**kwargs)
return items
def delete(self):
App.db().delete(self)
App.db().commit()
App.db().close()
def update(self):
App.db().merge(self)
App.db().commit()
App.db().close()
def insert(self):
App.db().add(self)
App.db().commit()
App.db().close()
def generate_dashboard(self, max_failures=MAX_FAILURES):
"""
Generate page with metrics indicating configuration of tx-manager.
:param int max_failures:
"""
App.logger.debug("Start: generateDashboard")
dashboard = {
'title': 'tX-Manager Dashboard',
'body': 'No modules found'
}
items = sorted(TxModule().query(), key=lambda k: k.name)
if items and len(items):
module_names = []
for item in items:
module_names.append(item.name)
App.logger.debug("Found: " + str(len(items)) + " item[s] in tx-module")
App.logger.debug("Reading from Jobs table")
registered_jobs = self.list_jobs({"convert_module": {"condition": "is_in", "value": module_names}}, False)
total_job_count = TxJob.query().count()
registered_job_count = registered_jobs.count()
App.logger.debug("Finished reading from Jobs table")
# sanity check since AWS can be slow to update job count reported in table (every 6 hours)
if registered_job_count > total_job_count:
total_job_count = registered_job_count
body = BeautifulSoup('<h1>TX-Manager Dashboard - {0}</h1>'
'<h2>Module Attributes</h2><br><table id="status"></table>'.format(datetime.now()),
'html.parser')
for item in items:
module_name = item.name
App.logger.debug(module_name)
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '"><td class="hdr" colspan="2">' + str(module_name) + '</td></tr>',
'html.parser'))
self.get_jobs_counts_for_module(registered_jobs, module_name)
# TBD the following code almosts walks the db record replacing next 11 lines
# for attr, val in item:
# if (attr != 'name') and (len(attr) > 0):
# rec += ' <tr><td class="lbl">' + attr.replace("_", " ").title() + ':</td><td>' + "lst(val)" + "</td></tr>\n"
# rec += '<tr><td colspan="2"></td></tr>'
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-type" class="module-type"><td class="lbl">Type:</td><td>' +
str(item.type) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-input" class="module-input"><td class="lbl">Input Format:</td><td>' +
json.dumps(item.input_format) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-output" class="module-output">' +
'<td class="lbl">Output Format:</td><td>' +
json.dumps(item.output_format) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-resource" class="module-resource"><td class="lbl">Resource Types:</td>'
'<td>' + json.dumps(item.resource_types) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-version" class="module-version"><td class="lbl">Version:</td><td>' +
str(item.version) + '</td></tr>',
'html.parser'))
if len(item.options) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-options" class="module-options">' +
'<td class="lbl">Options:</td><td>' +
json.dumps(item.options) + '</td></tr>',
'html.parser'))
if len(item.private_links) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-private-links" class="module-private-links">' +
'<td class="lbl">Private Links:</td><td>' +
json.dumps(item.private_links) + '</td></tr>',
'html.parser'))
if len(item.public_links) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-public-links" class="module-public-links">' +
'<td class="lbl">Public Links:</td><td>' +
json.dumps(item.public_links) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-success" class="module-public-links">' +
'<td class="lbl">Job Successes:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-warning" class="module-public-links">' +
'<td class="lbl">Job Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-failure" class="module-public-links">' +
'<td class="lbl">Job Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-total" class="module-public-links">' +
'<td class="lbl">Jobs Total:</td><td>' +
str(self.jobs_total) + '</td></tr>',
'html.parser'))
self.get_jobs_counts(registered_jobs)
body.table.append(BeautifulSoup(
'<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-success" class="module-public-links"><td class="lbl">Success:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-warning" class="module-public-links"><td class="lbl">Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-unregistered" class="module-public-links"><td class="lbl">Unregistered:</td><td>' +
str(total_job_count - self.jobs_total) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' +
str(total_job_count) + '</td></tr>',
'html.parser'))
# build job failures table
job_failures = self.get_job_failures(registered_jobs, max_failures)
body.append(BeautifulSoup('<h2>Failed Jobs</h2>', 'html.parser'))
failure_table = BeautifulSoup('<table id="failed" cellpadding="4" border="1" ' +
'style="border-collapse:collapse"></table>', 'html.parser')
failure_table.table.append(BeautifulSoup('''
<tr id="header">
<th class="hdr">Time</th>
<th class="hdr">Errors</th>
<th class="hdr">Repo</th>
<th class="hdr">PreConvert</th>
<th class="hdr">Converted</th>
<th class="hdr">Destination</th>''', 'html.parser'))
gogs_url = App.gogs_url
if gogs_url is None:
gogs_url = 'https://git.door43.org'
for i in range(0, len(job_failures)):
item = job_failures[i]
try:
identifier = item.identifier
user_name, repo_name, commit_id = identifier.split('/')[:3]
source_sub_path = '{0}/{1}'.format(user_name, repo_name)
cdn_bucket = item.cdn_bucket
destination_url = 'https://{0}/u/{1}/{2}/{3}/build_log.json'.format(cdn_bucket, user_name,
repo_name, commit_id)
repo_url = gogs_url + "/" + source_sub_path
preconverted_url = item.source
converted_url = item.output
failure_table.table.append(BeautifulSoup(
'<tr id="failure-' + str(i) + '" class="module-job-id">'
+ '<td>' + item.created_at.strftime("%Y-%m-%dT%H:%M:%SZ") + '</td>'
+ '<td>' + ','.join(item.errors) + '</td>'
+ '<td><a href="' + repo_url + '">' + source_sub_path + '</a></td>'
+ '<td><a href="' + preconverted_url + '">' + preconverted_url.rsplit('/', 1)[1] + '</a></td>'
+ '<td><a href="' + converted_url + '">' + item.job_id + '.zip</a></td>'
+ '<td><a href="' + destination_url + '">Build Log</a></td>'
+ '</tr>',
'html.parser'))
except Exception as e:
pass
body.append(failure_table)
self.build_language_popularity_tables(body, max_failures)
body_html = body.prettify('UTF-8')
dashboard['body'] = body_html
# save to cdn in case HTTP connection times out
try:
self.temp_dir = tempfile.mkdtemp(suffix="", prefix="dashboard_")
temp_file = os.path.join(self.temp_dir, "index.html")
file_utils.write_file(temp_file, body_html)
cdn_handler = App.cdn_s3_handler()
cdn_handler.upload_file(temp_file, 'dashboard/index.html')
except Exception as e:
App.logger.debug("Could not save dashboard: " + str(e))
else:
App.logger.debug("No modules found.")
App.db().close()
return dashboard
