def test_strftime_iso8601(self):
dt = datetime.datetime(year=2000, month=1, day=1, hour=12, minute=0, second=0)
dt = eva.coerce_to_utc(dt)
s = eva.strftime_iso8601(dt)
self.assertEqual(s, '2000-01-01T12:00:00+0000')
dt = 'foo'
with self.assertRaises(AttributeError):
eva.strftime_iso8601(dt)
s = eva.strftime_iso8601(dt, null_string=True)
self.assertEqual(s, 'NULL')
def test_create_job(self):
"""!
@brief Test that job creation generates the correct GridPP command line.
"""
self.create_adapter()
resource = mock.MagicMock()
resource.url = 'file:///foo/bar/baz'
resource.data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2016, 1, 1, 18, 15, 0))
with httmock.HTTMock(*eva.tests.schemas.SCHEMAS):
job = self.create_job(resource)
self.assertTrue('cp -v /foo/bar/baz /out/20160101T181500Z' in job.command)
self.assertTrue('export OMP_NUM_THREADS=4' in job.command)
self.assertTrue('gridpp /foo/bar/baz inopts /out/20160101T181500Z outopts --reftime 20160101T181500Z' in job.command)
def test_create_job(self):
"""!
@brief Test that job creation generates the correct FIMEX command line.
"""
self.create_adapter()
resource = mock.MagicMock()
resource.url = 'file:///foo/bar/baz'
resource.data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2016, 1, 1, 18, 15, 0))
with httmock.HTTMock(*eva.tests.schemas.SCHEMAS):
job = self.create_job(resource)
print(job.command)
self.assertTrue("[ ! -f '/output/20160101T181500Z.nc' ] && cp -v '/template/20160101T181500Z.nc' '/output/20160101T181500Z.nc'" in job.command)
self.assertTrue("fimex --input.file '/foo/bar/baz' --output.fillFile '/output/20160101T181500Z.nc'" in job.command)
def test_create_job(self):
"""!
@brief Test that job creation generates the correct FIMEX command line.
"""
self.create_adapter()
resource = mock.MagicMock()
resource.url = "file:///foo/bar/baz"
resource.data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2016, 1, 1, 18, 15, 0))
with httmock.HTTMock(*eva.tests.schemas.SCHEMAS):
job = self.create_job(resource)
command_line_fragment = (
"fimex --input.file '/foo/bar/baz' --output.file '/foo/bar.nc' 20160101T181500Z file:///foo/bar/baz baz\n"
)
self.assertTrue(command_line_fragment in job.command)
def test_create_job(self):
"""!
@brief Test that job creation generates the correct FIMEX command line.
"""
self.create_adapter()
resource = mock.MagicMock()
resource.url = 'file:///foo/bar/baz'
resource.data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2016, 1, 1, 18, 15, 0))
with httmock.HTTMock(*eva.tests.schemas.SCHEMAS):
job = self.create_job(resource)
self.assertTrue('/lib_fg2nc/grib2nc' in job.command)
self.assertTrue('--input "/foo/bar/baz"' in job.command)
self.assertTrue('--output "/new/20160101T181500Z/foo"' in job.command)
self.assertTrue('--reference_time "2016-01-01T18:15:00+0000"' in job.command)
self.assertTrue('--template_directory "/lib_fg2nc/template"' in job.command)
def test_finish_job_and_generate_resources(self):
"""!
@brief Test that job finish works and doesn't throw any exceptions.
"""
self.create_adapter()
resource = mock.MagicMock()
resource.url = 'file:///foo/bar/baz'
resource.data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2016, 1, 1, 18, 15, 0))
with httmock.HTTMock(*eva.tests.schemas.SCHEMAS):
job = self.create_job(resource)
job.set_status(eva.job.COMPLETE)
self.adapter.finish_job(job)
resources = self.generate_resources(job)
self.assertEqual(len(resources['productinstance']), 1)
self.assertEqual(len(resources['data']), 1)
self.assertEqual(len(resources['datainstance']), 1)
self.assertEqual(resources['datainstance'][0].url, 'file:///out/20160101T181500Z')
def test_sort_queue_adaptive(self):
now = eva.now_with_timezone()
# create six mock objects
mocks = [mock.MagicMock() for x in range(6)]
# set type to 'datainstance' for the first four objects, and set reference time to now
for x in [0, 1, 2, 3]:
mocks[x]._collection._resource_name = 'datainstance'
mocks[x].data.productinstance.reference_time = now
# set future reference time for the last two objects datainstance objects
for x in [2, 3]:
mocks[x].data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2100, 1, 1, 12, 0, 0))
# set event queue contents to two ProductstatusResourceEvents and
# three ProductstatusLocalEvents, data pointing to the mock objects,
# with increasing timestamps
self.eventloop.event_queue = [
eva.event.ProductstatusLocalEvent(str(x), mocks[x], timestamp=now + datetime.timedelta(hours=x + 1))
for x in [0, 1, 4]
]
self.eventloop.event_queue += [
eva.event.ProductstatusResourceEvent(str(x), mocks[x], timestamp=now + datetime.timedelta(hours=x + 1))
for x in [2, 3]
]
# add an rpc event to the event queue
self.eventloop.event_queue += [eva.event.RPCEvent(len(mocks) - 1, mocks[-1], timestamp=now + datetime.timedelta(hours=12))]
# set queue sort order
self.eventloop.queue_order = self.eventloop.QUEUE_ORDER_ADAPTIVE
# sort queue
logging.info('=== PRE-SORT ===')
[logging.info("%d: ts=%s type=%s reference_time=%s", i, x.timestamp(), x.__class__.__name__, x.data.data.productinstance.reference_time) for i, x in enumerate(self.eventloop.event_queue)]
self.eventloop.sort_queue()
logging.info('=== POST-SORT ===')
[logging.info("%d: ts=%s type=%s reference_time=%s", i, x.timestamp(), x.__class__.__name__, x.data.data.productinstance.reference_time) for i, x in enumerate(self.eventloop.event_queue)]
# test expected order: 1 rpc event first, then 2 objects with future
# reftimes, then 2 objects with "now" reference time, then 1
# non-datainstance object
expected_order = [5, 2, 3, 0, 1, 4]
real_order = [int(x.message) for x in self.eventloop.event_queue]
for i, n in enumerate(expected_order):
self.assertEqual(self.eventloop.event_queue[i].data, mocks[n],
msg='Queue was sorted incorrectly. Expected %s but got %s' % (expected_order, real_order))
def test_coerce_to_utc(self):
dt = datetime.datetime(year=2000, month=1, day=1, hour=12, minute=0, second=0)
dt_c = eva.coerce_to_utc(dt)
self.assertEqual(dt_c.tzinfo.tzname(None), 'UTC')
def test_productstatus_event_factory_unrecognized(self):
event = eva.event.ProductstatusBaseEvent.factory(unrecognized_serialized_event)
self.assertIsInstance(event, eva.event.ProductstatusBaseEvent)
self.assertEqual(event.id(), '20000000-0000-0000-0000-000000000002')
self.assertEqual(event.timestamp(), eva.coerce_to_utc(datetime.datetime(2016, 1, 5, 12, 0, 0)))
self.assertEqual(event.protocol_version(), [1, 5, 0])
