def test_load_module(self):
# Test loading by just giving it the name in the constructor
tx_module = TxModule.get(name='module1')
self.assertEqual(tx_module.resource_types, self.items['module1']['resource_types'])
# Test loading by just giving it only the name in the data array in the constructor
tx_module = TxModule.get(name='module2')
self.assertEqual(tx_module.input_format, self.items['module2']['input_format'])
def populate_tables(self):
for idx in self.job_items:
tx_job = TxJob(**self.job_items[idx])
tx_job.insert()
for idx in self.module_items:
tx_module = TxModule(**self.module_items[idx])
tx_module.insert()
def test_load_module(self):
# Test loading by just giving it the name in the constructor
tx_module = TxModule.get(name='module1')
self.assertEqual(tx_module.resource_types,
self.items['module1']['resource_types'])
# Test loading by just giving it only the name in the data array in the constructor
tx_module = TxModule.get(name='module2')
self.assertEqual(tx_module.input_format,
self.items['module2']['input_format'])
def test_register_module(self):
data = {
'name': 'module4',
'type': 'conversion',
'resource_types': ['obs'],
'input_format': 'md',
'output_format': 'html',
'options': {'pageSize': 'A4'},
'public_links': [],
'private_links': []
}
self.tx_manager.register_module(data)
tx_module = TxModule.get(name=data['name'])
self.assertIsNotNone(tx_module)
self.assertEqual(tx_module.options['pageSize'], 'A4')
self.assertEqual(tx_module.created_at.year, datetime.utcnow().year)
self.assertEqual(tx_module.updated_at.year, datetime.utcnow().year)
self.assertEqual(tx_module.public_links, ['{0}/tx/convert/{1}'.format(App.api_url, data['name'])])
test_missing_keys = ['name', 'type', 'input_format', 'resource_types']
for key in test_missing_keys:
# should raise an exception if data is missing a required field
missing = data.copy()
del missing[key]
self.assertRaises(Exception, self.tx_manager.register_module, missing)
def query_linters(self, resource_type=None, input_format=None):
query = TxModule.query().filter(TxModule.type == 'linter')
if input_format:
query = query.filter(TxModule.input_format.contains(input_format))
if resource_type:
query = query.filter(
TxModule.resource_types.contains(resource_type))
return query
def register_module(data):
tx_module = TxModule(**data)
if not tx_module.name:
raise Exception('"name" not given.')
if not tx_module.type:
raise Exception('"type" not given.')
if not tx_module.input_format:
raise Exception('"input_format" not given.')
if not tx_module.resource_types:
raise Exception('"resource_types" not given.')
tx_module.public_links.append("{0}/tx/convert/{1}".format(App.api_url, tx_module.name))
old_module = TxModule.get(name=tx_module.name)
if old_module:
old_module.delete()
tx_module.insert()
return tx_module
def get_linter_module(self, job):
"""
:param TxJob job:
:return TxModule:
"""
linters = TxModule.query().filter(TxModule.type=='linter') \
.filter(TxModule.input_format.contains(job.input_format))
linter = linters.filter(TxModule.resource_types.contains(job.resource_type)).first()
if not linter:
linter = linters.filter(TxModule.resource_types.contains('other')).first()
return linter
def get_linter_module(self, job):
"""
:param TxJob job:
:return TxModule:
"""
linters = TxModule.query().filter(TxModule.type=='linter') \
.filter(TxModule.input_format.contains(job.input_format))
linter = linters.filter(TxModule.resource_types.contains(job.resource_type)).first()
if not linter:
linter = linters.filter(TxModule.resource_types.contains('other')).first()
return linter
def get_converter_module(self, job):
"""
:param TxJob job:
:return TxModule:
"""
converters = TxModule.query().filter(TxModule.type=='converter') \
.filter(TxModule.input_format.contains(job.input_format)) \
.filter(TxModule.output_format.contains(job.output_format))
converter = converters.filter(TxModule.resource_types.contains(job.resource_type)).first()
if not converter:
converter = converters.filter(TxModule.resource_types.contains('other')).first()
return converter
def get_converter_module(self, job):
"""
:param TxJob job:
:return TxModule:
"""
converters = TxModule.query().filter(TxModule.type=='converter') \
.filter(TxModule.input_format.contains(job.input_format)) \
.filter(TxModule.output_format.contains(job.output_format))
converter = converters.filter(TxModule.resource_types.contains(job.resource_type)).first()
if not converter:
converter = converters.filter(TxModule.resource_types.contains('other')).first()
return converter
def test_handle(self, mock_register_module):
mock_register_module.return_value = TxModule()
event = {
'data': {
"name": "tx-md2html_convert",
"version": "1",
"type": "conversion",
"resource_types": ["obs"],
"input_format": ["md"],
"output_format": ["html"],
"options": {'pageSize': 'A4'},
"private_links": [],
"public_links": []
}
}
handler = RegisterModuleHandler()
self.assertIsInstance(handler.handle(event, None), dict)
def test_update_module(self):
tx_module = TxModule.get(name=self.items['module3']['name'])
tx_module.output_format = 'usfm'
tx_module.update()
tx_module = TxModule.get(name=self.items['module3']['name'])
self.assertEqual(tx_module.output_format, 'usfm')
def test_delete_module(self):
tx_module = TxModule.get(name=self.items['module1']['name'])
self.assertIsNotNone(tx_module)
tx_module.delete()
tx_module = TxModule.get(name=self.items['module1']['name'])
self.assertIsNone(tx_module)
def test_query_module(self):
tx_modules = TxModule.query()
self.assertEqual(tx_modules.count(), len(self.items))
for tx_module in tx_modules:
self.assertEqual(tx_module.resource_types,
self.items[tx_module.name]['resource_types'])
def generate_dashboard(self, max_failures=MAX_FAILURES):
"""
Generate page with metrics indicating configuration of tx-manager.
:param int max_failures:
"""
App.logger.debug("Start: generateDashboard")
dashboard = {
'title': 'tX-Manager Dashboard',
'body': 'No modules found'
}
items = sorted(TxModule().query(), key=lambda k: k.name)
if items and len(items):
module_names = []
for item in items:
module_names.append(item.name)
App.logger.debug("Found: " + str(len(items)) + " item[s] in tx-module")
App.logger.debug("Reading from Jobs table")
registered_jobs = self.list_jobs({"convert_module": {"condition": "is_in", "value": module_names}}, False)
total_job_count = TxJob.query().count()
registered_job_count = registered_jobs.count()
App.logger.debug("Finished reading from Jobs table")
# sanity check since AWS can be slow to update job count reported in table (every 6 hours)
if registered_job_count > total_job_count:
total_job_count = registered_job_count
body = BeautifulSoup('<h1>TX-Manager Dashboard - {0}</h1>'
'<h2>Module Attributes</h2><br><table id="status"></table>'.format(datetime.now()),
'html.parser')
for item in items:
module_name = item.name
App.logger.debug(module_name)
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '"><td class="hdr" colspan="2">' + str(module_name) + '</td></tr>',
'html.parser'))
self.get_jobs_counts_for_module(registered_jobs, module_name)
# TBD the following code almosts walks the db record replacing next 11 lines
# for attr, val in item:
# if (attr != 'name') and (len(attr) > 0):
# rec += ' <tr><td class="lbl">' + attr.replace("_", " ").title() + ':</td><td>' + "lst(val)" + "</td></tr>\n"
# rec += '<tr><td colspan="2"></td></tr>'
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-type" class="module-type"><td class="lbl">Type:</td><td>' +
str(item.type) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-input" class="module-input"><td class="lbl">Input Format:</td><td>' +
json.dumps(item.input_format) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-output" class="module-output">' +
'<td class="lbl">Output Format:</td><td>' +
json.dumps(item.output_format) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-resource" class="module-resource"><td class="lbl">Resource Types:</td>'
'<td>' + json.dumps(item.resource_types) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-version" class="module-version"><td class="lbl">Version:</td><td>' +
str(item.version) + '</td></tr>',
'html.parser'))
if len(item.options) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-options" class="module-options">' +
'<td class="lbl">Options:</td><td>' +
json.dumps(item.options) + '</td></tr>',
'html.parser'))
if len(item.private_links) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-private-links" class="module-private-links">' +
'<td class="lbl">Private Links:</td><td>' +
json.dumps(item.private_links) + '</td></tr>',
'html.parser'))
if len(item.public_links) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-public-links" class="module-public-links">' +
'<td class="lbl">Public Links:</td><td>' +
json.dumps(item.public_links) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-success" class="module-public-links">' +
'<td class="lbl">Job Successes:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-warning" class="module-public-links">' +
'<td class="lbl">Job Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-failure" class="module-public-links">' +
'<td class="lbl">Job Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + module_name + '-job-total" class="module-public-links">' +
'<td class="lbl">Jobs Total:</td><td>' +
str(self.jobs_total) + '</td></tr>',
'html.parser'))
self.get_jobs_counts(registered_jobs)
body.table.append(BeautifulSoup(
'<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-success" class="module-public-links"><td class="lbl">Success:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-warning" class="module-public-links"><td class="lbl">Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-unregistered" class="module-public-links"><td class="lbl">Unregistered:</td><td>' +
str(total_job_count - self.jobs_total) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' +
str(total_job_count) + '</td></tr>',
'html.parser'))
# build job failures table
job_failures = self.get_job_failures(registered_jobs, max_failures)
body.append(BeautifulSoup('<h2>Failed Jobs</h2>', 'html.parser'))
failure_table = BeautifulSoup('<table id="failed" cellpadding="4" border="1" ' +
'style="border-collapse:collapse"></table>', 'html.parser')
failure_table.table.append(BeautifulSoup('''
<tr id="header">
<th class="hdr">Time</th>
<th class="hdr">Errors</th>
<th class="hdr">Repo</th>
<th class="hdr">PreConvert</th>
<th class="hdr">Converted</th>
<th class="hdr">Destination</th>''', 'html.parser'))
gogs_url = App.gogs_url
if gogs_url is None:
gogs_url = 'https://git.door43.org'
for i in range(0, len(job_failures)):
item = job_failures[i]
try:
identifier = item.identifier
user_name, repo_name, commit_id = identifier.split('/')[:3]
source_sub_path = '{0}/{1}'.format(user_name, repo_name)
cdn_bucket = item.cdn_bucket
destination_url = 'https://{0}/u/{1}/{2}/{3}/build_log.json'.format(cdn_bucket, user_name,
repo_name, commit_id)
repo_url = gogs_url + "/" + source_sub_path
preconverted_url = item.source
converted_url = item.output
failure_table.table.append(BeautifulSoup(
'<tr id="failure-' + str(i) + '" class="module-job-id">'
+ '<td>' + item.created_at.strftime("%Y-%m-%dT%H:%M:%SZ") + '</td>'
+ '<td>' + ','.join(item.errors) + '</td>'
+ '<td><a href="' + repo_url + '">' + source_sub_path + '</a></td>'
+ '<td><a href="' + preconverted_url + '">' + preconverted_url.rsplit('/', 1)[1] + '</a></td>'
+ '<td><a href="' + converted_url + '">' + item.job_id + '.zip</a></td>'
+ '<td><a href="' + destination_url + '">Build Log</a></td>'
+ '</tr>',
'html.parser'))
except Exception as e:
pass
body.append(failure_table)
self.build_language_popularity_tables(body, max_failures)
body_html = body.prettify('UTF-8')
dashboard['body'] = body_html
# save to cdn in case HTTP connection times out
try:
self.temp_dir = tempfile.mkdtemp(suffix="", prefix="dashboard_")
temp_file = os.path.join(self.temp_dir, "index.html")
file_utils.write_file(temp_file, body_html)
cdn_handler = App.cdn_s3_handler()
cdn_handler.upload_file(temp_file, 'dashboard/index.html')
except Exception as e:
App.logger.debug("Could not save dashboard: " + str(e))
else:
App.logger.debug("No modules found.")
App.db().close()
return dashboard
def populate_table(self):
for idx in self.items:
tx_module = TxModule(**self.items[idx])
tx_module.insert()
def populate_table(self):
for idx in self.items:
tx_module = TxModule(**self.items[idx])
tx_module.insert()
def test_delete_module(self):
tx_module = TxModule.get(name=self.items['module1']['name'])
self.assertIsNotNone(tx_module)
tx_module.delete()
tx_module = TxModule.get(name=self.items['module1']['name'])
self.assertIsNone(tx_module)
def test_update_module(self):
tx_module = TxModule.get(name=self.items['module3']['name'])
tx_module.output_format = 'usfm'
tx_module.update()
tx_module = TxModule.get(name=self.items['module3']['name'])
self.assertEqual(tx_module.output_format, 'usfm')
def test_query_module(self):
tx_modules = TxModule.query()
self.assertEqual(tx_modules.count(), len(self.items))
for tx_module in tx_modules:
self.assertEqual(tx_module.resource_types, self.items[tx_module.name]['resource_types'])