def get(cls, *args, **kwargs): """ :param args: :param kwargs: :return TxModel: """ if args: kwargs[inspect(cls).primary_key[0].name] = args[0] item = cls.query(**kwargs).first() App.db().close() return item
def get(cls, *args, **kwargs): """ :param args: :param kwargs: :return TxModel: """ if args: kwargs[inspect(cls).primary_key[0].name] = args[0] item = cls.query(**kwargs).first() App.db().close() return item
def query(cls, **kwargs): items = App.db().query(cls).filter_by(**kwargs) return items
def delete(self): App.db().delete(self) App.db().commit() App.db().close()
def update(self): App.db().merge(self) App.db().commit() App.db().close()
def insert(self): App.db().add(self) App.db().commit() App.db().close()
def query(cls, **kwargs): items = App.db().query(cls).filter_by(**kwargs) return items
def delete(self): App.db().delete(self) App.db().commit() App.db().close()
def update(self): App.db().merge(self) App.db().commit() App.db().close()
def insert(self): App.db().add(self) App.db().commit() App.db().close()
def generate_dashboard(self, max_failures=MAX_FAILURES): """ Generate page with metrics indicating configuration of tx-manager. :param int max_failures: """ App.logger.debug("Start: generateDashboard") dashboard = { 'title': 'tX-Manager Dashboard', 'body': 'No modules found' } items = sorted(TxModule().query(), key=lambda k: k.name) if items and len(items): module_names = [] for item in items: module_names.append(item.name) App.logger.debug("Found: " + str(len(items)) + " item[s] in tx-module") App.logger.debug("Reading from Jobs table") registered_jobs = self.list_jobs({"convert_module": {"condition": "is_in", "value": module_names}}, False) total_job_count = TxJob.query().count() registered_job_count = registered_jobs.count() App.logger.debug("Finished reading from Jobs table") # sanity check since AWS can be slow to update job count reported in table (every 6 hours) if registered_job_count > total_job_count: total_job_count = registered_job_count body = BeautifulSoup('<h1>TX-Manager Dashboard - {0}</h1>' '<h2>Module Attributes</h2><br><table id="status"></table>'.format(datetime.now()), 'html.parser') for item in items: module_name = item.name App.logger.debug(module_name) body.table.append(BeautifulSoup( '<tr id="' + module_name + '"><td class="hdr" colspan="2">' + str(module_name) + '</td></tr>', 'html.parser')) self.get_jobs_counts_for_module(registered_jobs, module_name) # TBD the following code almosts walks the db record replacing next 11 lines # for attr, val in item: # if (attr != 'name') and (len(attr) > 0): # rec += ' <tr><td class="lbl">' + attr.replace("_", " ").title() + ':</td><td>' + "lst(val)" + "</td></tr>\n" # rec += '<tr><td colspan="2"></td></tr>' body.table.append(BeautifulSoup( '<tr id="' + module_name + '-type" class="module-type"><td class="lbl">Type:</td><td>' + str(item.type) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-input" class="module-input"><td class="lbl">Input Format:</td><td>' + json.dumps(item.input_format) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-output" class="module-output">' + '<td class="lbl">Output Format:</td><td>' + json.dumps(item.output_format) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-resource" class="module-resource"><td class="lbl">Resource Types:</td>' '<td>' + json.dumps(item.resource_types) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-version" class="module-version"><td class="lbl">Version:</td><td>' + str(item.version) + '</td></tr>', 'html.parser')) if len(item.options) > 0: body.table.append(BeautifulSoup( '<tr id="' + module_name + '-options" class="module-options">' + '<td class="lbl">Options:</td><td>' + json.dumps(item.options) + '</td></tr>', 'html.parser')) if len(item.private_links) > 0: body.table.append(BeautifulSoup( '<tr id="' + module_name + '-private-links" class="module-private-links">' + '<td class="lbl">Private Links:</td><td>' + json.dumps(item.private_links) + '</td></tr>', 'html.parser')) if len(item.public_links) > 0: body.table.append(BeautifulSoup( '<tr id="' + module_name + '-public-links" class="module-public-links">' + '<td class="lbl">Public Links:</td><td>' + json.dumps(item.public_links) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-job-success" class="module-public-links">' + '<td class="lbl">Job Successes:</td><td>' + str(self.jobs_success) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-job-warning" class="module-public-links">' + '<td class="lbl">Job Warnings:</td><td>' + str(self.jobs_warnings) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-job-failure" class="module-public-links">' + '<td class="lbl">Job Failures:</td><td>' + str(self.jobs_failures) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="' + module_name + '-job-total" class="module-public-links">' + '<td class="lbl">Jobs Total:</td><td>' + str(self.jobs_total) + '</td></tr>', 'html.parser')) self.get_jobs_counts(registered_jobs) body.table.append(BeautifulSoup( '<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="totals-job-success" class="module-public-links"><td class="lbl">Success:</td><td>' + str(self.jobs_success) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="totals-job-warning" class="module-public-links"><td class="lbl">Warnings:</td><td>' + str(self.jobs_warnings) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' + str(self.jobs_failures) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="totals-job-unregistered" class="module-public-links"><td class="lbl">Unregistered:</td><td>' + str(total_job_count - self.jobs_total) + '</td></tr>', 'html.parser')) body.table.append(BeautifulSoup( '<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' + str(total_job_count) + '</td></tr>', 'html.parser')) # build job failures table job_failures = self.get_job_failures(registered_jobs, max_failures) body.append(BeautifulSoup('<h2>Failed Jobs</h2>', 'html.parser')) failure_table = BeautifulSoup('<table id="failed" cellpadding="4" border="1" ' + 'style="border-collapse:collapse"></table>', 'html.parser') failure_table.table.append(BeautifulSoup(''' <tr id="header"> <th class="hdr">Time</th> <th class="hdr">Errors</th> <th class="hdr">Repo</th> <th class="hdr">PreConvert</th> <th class="hdr">Converted</th> <th class="hdr">Destination</th>''', 'html.parser')) gogs_url = App.gogs_url if gogs_url is None: gogs_url = 'https://git.door43.org' for i in range(0, len(job_failures)): item = job_failures[i] try: identifier = item.identifier user_name, repo_name, commit_id = identifier.split('/')[:3] source_sub_path = '{0}/{1}'.format(user_name, repo_name) cdn_bucket = item.cdn_bucket destination_url = 'https://{0}/u/{1}/{2}/{3}/build_log.json'.format(cdn_bucket, user_name, repo_name, commit_id) repo_url = gogs_url + "/" + source_sub_path preconverted_url = item.source converted_url = item.output failure_table.table.append(BeautifulSoup( '<tr id="failure-' + str(i) + '" class="module-job-id">' + '<td>' + item.created_at.strftime("%Y-%m-%dT%H:%M:%SZ") + '</td>' + '<td>' + ','.join(item.errors) + '</td>' + '<td><a href="' + repo_url + '">' + source_sub_path + '</a></td>' + '<td><a href="' + preconverted_url + '">' + preconverted_url.rsplit('/', 1)[1] + '</a></td>' + '<td><a href="' + converted_url + '">' + item.job_id + '.zip</a></td>' + '<td><a href="' + destination_url + '">Build Log</a></td>' + '</tr>', 'html.parser')) except Exception as e: pass body.append(failure_table) self.build_language_popularity_tables(body, max_failures) body_html = body.prettify('UTF-8') dashboard['body'] = body_html # save to cdn in case HTTP connection times out try: self.temp_dir = tempfile.mkdtemp(suffix="", prefix="dashboard_") temp_file = os.path.join(self.temp_dir, "index.html") file_utils.write_file(temp_file, body_html) cdn_handler = App.cdn_s3_handler() cdn_handler.upload_file(temp_file, 'dashboard/index.html') except Exception as e: App.logger.debug("Could not save dashboard: " + str(e)) else: App.logger.debug("No modules found.") App.db().close() return dashboard