def query(args, parser):
""" query command
The query subcommmand behavior is determined by the --pending-pages and
--avg-runtimes arguments.
When --pending-pages is passed the number of pending pages is printed. This query
can be modified via the --filters argument.
When --avg-runtimes is passed the log files from emop-controller are parsed and average runtimes
are printed.
"""
emop_query = EmopQuery(args.config_path)
# --pending-pages
if args.query_pending_pages:
pending_pages = emop_query.pending_pages_count(q_filter=args.filter)
if pending_pages == 0 or pending_pages:
print("Number of pending pages: %s" % pending_pages)
else:
print("ERROR: querying pending pages failed")
sys.exit(1)
# --avg-runtimes
if args.query_avg_runtimes:
avg_runtimes = emop_query.get_runtimes()
if avg_runtimes:
print("Pages completed: %d" % avg_runtimes["total_pages"])
print("Total Page Runtime: %d seconds" % avg_runtimes["total_page_runtime"])
print("Average Page Runtime: %d seconds" % avg_runtimes["average_page_runtime"])
print("Jobs completed: %d" % avg_runtimes["total_jobs"])
print("Average Job Runtime: %d seconds" % avg_runtimes["average_job_runtime"])
print("Processes:")
for process in avg_runtimes["processes"]:
print("\t%s completed: %d" % (process["name"], process["count"]))
print("\t%s Average: %d seconds" % (process["name"], process["avg"]))
print("\t%s Total: %d seconds" % (process["name"], process["total"]))
else:
print("ERROR: querying average page runtimes")
sys.exit(1)
sys.exit(0)
class TestEmopQuery(TestCase):
def setUp(self):
self.query = EmopQuery(config_path=default_config_path())
def tearDown(self):
pass
def test_get_job_status_id(self):
mock_response = {
"total": 1,
"subtotal": 1,
"page": 1,
"per_page": 1,
"total_pages": 1,
"results": [
{
"id": 1,
"name": "Not Started"
},
]
}
self.query.emop_api.get_request = MagicMock()
self.query.emop_api.get_request.return_value = mock_response
retval = self.query._get_job_status_id()
self.assertEqual(1, retval)
def test_pending_pages_count(self):
mock_response = {
"job_queue": {
"count": 2
}
}
self.query._get_job_status_id = MagicMock()
self.query._get_job_status_id.return_value = 1
self.query.emop_api.get_request = MagicMock()
self.query.emop_api.get_request.return_value = mock_response
retval = self.query.pending_pages_count(q_filter='{"batch_id": 1}')
self.assertEqual(2, retval)
def test_pending_pages_1(self):
mock_response = load_fixture_file('job_queues_1.json')
self.query._get_job_status_id = MagicMock()
self.query._get_job_status_id.return_value = 1
self.query.emop_api.get_request = MagicMock()
self.query.emop_api.get_request.return_value = mock_response
retval = self.query.pending_pages(q_filter='{"batch_id": 1}')
self.assertEqual(mock_response['results'], retval)
@xfail
def test_pending_pages_2(self):
mock_response = load_fixture_file('job_queues_1.json')
expected = [
{
'page': {
"pg_ground_truth_file": "/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/1.txt",
'pg_image_path': '/data/eebo/e0006/40099/00001.000.001.tif',
},
},
{
'page': {
"pg_ground_truth_file": "/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/2.txt",
'pg_image_path': '/data/eebo/e0006/40099/00002.000.001.tif',
},
},
]
self.query._get_job_status_id = MagicMock()
self.query._get_job_status_id.return_value = 1
self.query.emop_api.get_request = MagicMock()
self.query.emop_api.get_request.return_value = mock_response
retval = self.query.pending_pages(q_filter='{"batch_id": 1}', r_filter='page.pg_image_path,pg_ground_truth_file')
self.maxDiff = None
self.assertEqual(expected, retval)
def test_get_runtimes_1(self):
expected = {
'total_pages': 10,
'total_page_runtime': 630.943,
'average_page_runtime': 63.094,
'total_jobs': 1,
'average_job_runtime': 631.018,
'processes': [
{'name': "OCR", 'count': 10, 'total': 69.422,'avg': 6.942},
{'name': "Denoise", 'count': 10, 'total': 69.579,'avg': 6.958},
{'name': "MultiColumnSkew", 'count': 10, 'total': 69.331,'avg': 6.933},
{'name': "XML_To_Text", 'count': 10, 'total': 0.205,'avg': 0.021},
{'name': "PageEvaluator", 'count': 10, 'total': 9.852,'avg': 0.985},
{'name': "PageCorrector", 'count': 10, 'total': 402.345,'avg': 40.234},
{'name': "JuxtaCompare", 'count': 10, 'total': 10.118,'avg': 1.012},
],
}
self.query.settings.scheduler_logdir = os.path.dirname(fixture_file('log-1.out'))
retval = self.query.get_runtimes()
self.maxDiff = None
self.assertEqual(expected, retval)
