def test_stage_out_proc_id_1(self):
self.transfer.settings.payload_output_path = os.path.dirname(fixture_file('output_payload_1.json'))
payload = EmopPayload(self.transfer.settings, 'output_payload_1')
expected_data = [
{
'src': '/fdata/idhmc/emop-output/data/shared/text-xml/IDHMC-ocr/17/152141/1_ALTO.txt',
'dest': '/data/shared/text-xml/IDHMC-ocr/17/152141/1_ALTO.txt',
},
{
'src': '/fdata/idhmc/emop-output/data/shared/text-xml/IDHMC-ocr/17/152141/1.txt',
'dest': '/data/shared/text-xml/IDHMC-ocr/17/152141/1.txt',
},
{
'src': '/fdata/idhmc/emop-output/data/shared/text-xml/IDHMC-ocr/17/152141/1_ALTO.xml',
'dest': '/data/shared/text-xml/IDHMC-ocr/17/152141/1_ALTO.xml',
},
{
'src': '/fdata/idhmc/emop-output/data/shared/text-xml/IDHMC-ocr/17/152141/1.xml',
'dest': '/data/shared/text-xml/IDHMC-ocr/17/152141/1.xml',
}
]
self.transfer.start = MagicMock(return_value='000-000-001')
retval = self.transfer.stage_out_proc_id('output_payload_1')
self.transfer.start.assert_called_once_with(src='tamu#brazos', dest='idhmc#data', data=expected_data, label='emop-stage-out-output_payload_1', wait=False)
self.assertEqual('000-000-001', retval)
def test_get_runtimes_1(self):
expected = {
'total_pages': 10,
'total_page_runtime': 630.943,
'average_page_runtime': 63.094,
'total_jobs': 1,
'average_job_runtime': 631.018,
'processes': [
{'name': "OCR", 'count': 10, 'total': 69.422,'avg': 6.942},
{'name': "Denoise", 'count': 10, 'total': 69.579,'avg': 6.958},
{'name': "MultiColumnSkew", 'count': 10, 'total': 69.331,'avg': 6.933},
{'name': "XML_To_Text", 'count': 10, 'total': 0.205,'avg': 0.021},
{'name': "PageEvaluator", 'count': 10, 'total': 9.852,'avg': 0.985},
{'name': "PageCorrector", 'count': 10, 'total': 402.345,'avg': 40.234},
{'name': "JuxtaCompare", 'count': 10, 'total': 10.118,'avg': 1.012},
],
}
self.query.settings.scheduler_logdir = os.path.dirname(fixture_file('log-1.out'))
retval = self.query.get_runtimes()
self.maxDiff = None
self.assertEqual(expected, retval)
def test_stage_in_proc_ids(self):
self.transfer.settings.payload_input_path = os.path.dirname(fixture_file('input_payload_2.json'))
expected_data = [
{
'src': '/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/2.txt',
'dest': '/fdata/idhmc/emop-input/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/2.txt',
},
{
'src': '/data/eebo/e0006/40099/00001.000.001.tif',
'dest': '/fdata/idhmc/emop-input/data/eebo/e0006/40099/00001.000.001.tif',
},
{
'src': '/data/eebo/e0006/40099/00002.000.001.tif',
'dest': '/fdata/idhmc/emop-input/data/eebo/e0006/40099/00002.000.001.tif',
},
{
'src': '/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/1.txt',
'dest': '/fdata/idhmc/emop-input/data/shared/text-xml/EEBO-TCP-pages-text/e0006/40099/1.txt',
},
]
self.transfer.start = MagicMock()
self.transfer.stage_in_proc_ids(proc_ids=['input_payload_2'])
self.transfer.start.assert_called_once_with(src='idhmc#data', dest='tamu#brazos', data=expected_data, label='emop-stage-in-files', wait=False)
def test_stage_out_proc_id_4(self):
self.transfer.settings.payload_output_path = os.path.dirname(fixture_file('invalid.json'))
payload = EmopPayload(self.transfer.settings, 'invalid')
retval = self.transfer.stage_out_proc_id('invalid')
self.assertEqual('', retval)
