def test_sensor_pair_equality(self):
sensor_1 = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
sensor_2 = Sensor('atsr.2', Period((2007, 7, 1), (2008, 7, 1)))
sensor_pair_1 = SensorPair(sensor_1, sensor_2)
sensor_pair_2 = SensorPair(sensor_2, sensor_1)
self.assertTrue(sensor_pair_1 == sensor_pair_2)
def test_get_all_sensors_by_period(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
sensors = w._get_all_sensors_by_period()
""":type : list"""
self.assertEqual(3, len(sensors))
sensor_1 = sensors[0]
""":type : Sensor"""
sensor_2 = sensors[1]
""":type : Sensor"""
sensor_3 = sensors[2]
""":type : Sensor"""
self.assertEqual("avhrr.n10", sensor_1.get_name())
self.assertEqual(Period('1988-11-08', '1991-09-16'),
sensor_1.get_period())
self.assertEqual("avhrr.n11", sensor_2.get_name())
self.assertEqual(Period('1988-11-08', '1994-12-31'),
sensor_2.get_period())
self.assertEqual("avhrr.n12", sensor_3.get_name())
self.assertEqual(Period('1991-09-16', '1994-12-31'),
sensor_3.get_period())
def test_sensor_pair_construction(self):
sensor_1 = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
sensor_2 = Sensor('atsr.2', Period((2007, 7, 1), (2008, 7, 1)))
sensor_3 = Sensor('atsr.1', Period((2008, 1, 1), (2009, 1, 1)))
sensor_pair = SensorPair(sensor_1, sensor_2)
self.assertEqual('atsr.3', sensor_pair.get_primary_name())
self.assertEqual('atsr.2', sensor_pair.get_secondary_name())
self.assertEqual(Period((2007, 7, 1), (2008, 1, 1)),
sensor_pair.get_period())
sensor_pair = SensorPair(sensor_3, sensor_2)
self.assertEqual('atsr.1', sensor_pair.get_primary_name())
self.assertEqual('atsr.2', sensor_pair.get_secondary_name())
self.assertEqual(Period((2008, 1, 1), (2008, 7, 1)),
sensor_pair.get_period())
try:
SensorPair(sensor_1, sensor_3)
self.fail()
except exceptions.ValueError:
pass
try:
SensorPair(sensor_3, sensor_1)
except exceptions.ValueError:
pass
def test_sensor_ge(self):
sensor_1 = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
sensor_2 = Sensor('atsr.2', Period((2008, 1, 1), (2009, 1, 1)))
self.assertTrue(sensor_1 >= sensor_2)
sensor_3 = Sensor('atsr.2', Period((2009, 1, 1), (2010, 1, 1)))
self.assertTrue(sensor_2 >= sensor_3)
self.assertTrue(sensor_3 >= sensor_2)
def test_period_construction(self):
period_1 = Period((2007, 1, 1), '2008-01-01')
period_2 = Period('2007-01-01', (2008, 1, 1))
period_3 = Period((2007, 1, 1), datetime.date(2008, 1, 1))
period_4 = Period(datetime.date(2007, 1, 1), (2008, 1, 1))
self.assertTrue(period_1 == period_2)
self.assertTrue(period_2 == period_3)
self.assertTrue(period_3 == period_4)
self.assertTrue(period_4 == period_1)
def test_sensor_pair_ge(self):
sensor_1 = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
sensor_2 = Sensor('atsr.2', Period((2007, 7, 1), (2008, 7, 1)))
sensor_3 = Sensor('atsr.1', Period((2007, 10, 1), (2008, 10, 1)))
sensor_pair_1 = SensorPair(sensor_1, sensor_2)
sensor_pair_2 = SensorPair(sensor_2, sensor_3)
sensor_pair_3 = SensorPair(sensor_1, sensor_3)
self.assertTrue(sensor_pair_1 >= sensor_pair_2)
self.assertTrue(sensor_pair_1 >= sensor_pair_3)
def test_get_period_intersection(self):
period_1 = Period('2007-01-01', '2008-01-01')
period_2 = Period('2007-07-01', '2008-07-01')
period_3 = Period('2007-10-01', '2007-11-01')
period_4 = Period('2001-07-01', '2002-07-01')
self.assertEqual(period_1, period_1.get_intersection(period_1))
self.assertEqual(period_2, period_2.get_intersection(period_2))
self.assertEqual(Period('2007-07-01', '2008-01-01'), period_1.get_intersection(period_2))
self.assertEqual(Period('2007-07-01', '2008-01-01'), period_2.get_intersection(period_1))
self.assertEqual(period_3, period_1.get_intersection(period_3))
self.assertTrue(period_1.get_intersection(period_4) is None)
def add_sensor(self, name, start_date, end_date):
"""
:type name: str
"""
period = Period(start_date, end_date)
for sensor in self._get_sensors():
if sensor.get_name() == name and sensor.get_period(
).is_intersecting(period):
raise exceptions.ValueError, "Periods of sensor '" + name + "' must not intersect."
self.sensors.add(Sensor(name, period))
def test_get_inp_preconditions_for_one_month_minus_one_day(self):
w = Workflow('test', Period('1991-01-02', '1991-02-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = []
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(1, len(preconditions))
self.assertEqual('/inp/1991/01', preconditions[0])
def test_add_obs_preconditions_for_one_year_and_one_day(self):
w = Workflow('test', Period('1991-01-01', '1992-01-02'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_obs_preconditions(preconditions)
self.assertEqual(2, len(preconditions))
self.assertEqual('/obs/1990/12', preconditions[0])
self.assertEqual('/obs/1992/02', preconditions[1])
def test_get_sensors_by_period(self):
w = RegridWorkflow('test', '1.0', '.', '.')
w.add_sensor('AVHRR10_G', (1986, 11, 1), (1991, 9, 30))
w.add_sensor('AVHRR11_G', (1988, 11, 1), (1994, 12, 31))
w.add_sensor('AVHRR12_G', (1991, 9, 1), (1998, 12, 31))
sensors = w._get_sensors_by_period()
""":type : list"""
self.assertEqual(3, len(sensors))
sensor_1 = sensors[0]
""":type : Sensor"""
sensor_2 = sensors[1]
""":type : Sensor"""
sensor_3 = sensors[2]
""":type : Sensor"""
self.assertEqual("AVHRR10_G", sensor_1.get_name())
self.assertEqual(Period('1986-11-01', '1991-09-30'),
sensor_1.get_period())
self.assertEqual("AVHRR11_G", sensor_2.get_name())
self.assertEqual(Period('1988-11-01', '1994-12-31'),
sensor_2.get_period())
self.assertEqual("AVHRR12_G", sensor_3.get_name())
self.assertEqual(Period('1991-09-01', '1998-12-31'),
sensor_3.get_period())
def test_add_smp_preconditions_for_one_year(self):
w = Workflow('test', Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_smp_preconditions(preconditions)
self.assertEqual(4, len(preconditions))
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1991/08', preconditions[0])
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1992/01', preconditions[1])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1990/12', preconditions[2])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1991/10', preconditions[3])
def _get_data_period(self):
"""
:rtype : Period
"""
start_date = datetime.date.max
end_date = datetime.date.min
for sensor in self._get_sensors():
period = sensor.get_period()
if period.get_start_date() < start_date:
start_date = period.get_start_date()
if period.get_end_date() > end_date:
end_date = period.get_end_date()
if start_date < end_date:
return Period(start_date, end_date)
else:
return None
def test_run_single_sensor_usecase(self):
usecase = 'mms3'
mmdtype = 'mmd3'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', with_history=True, simulation=True)
with open('mms3.status', 'r') as status:
self.assertEqual(
'337 created, 0 running, 0 backlog, 337 processed, 0 failed\n',
status.readline())
with open('mms3.report', 'r') as report:
self.assertEqual(337, len(report.readlines()))
os.remove('mms3.status')
os.remove('mms3.report')
def test_run_dual_sensor_usecase_without_arc(self):
usecase = 'mms1'
mmdtype = 'mmd1'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', simulation=True, without_arc=True)
with open('mms1.status', 'r') as status:
self.assertEqual(
'203 created, 0 running, 0 backlog, 203 processed, 0 failed\n',
status.readline())
with open('mms1.report', 'r') as report:
self.assertEqual(203, len(report.readlines()))
os.remove('mms1.status')
os.remove('mms1.report')
def test_run_regrid_usecase(self):
usecase = 'regrid'
#version = 'v2.1.8'
archive_root = "/neodc/esacci_sst/data/lt/"
target_root = "/group_workspaces/cems2/esacci_sst/scratch/2015_05_regridded_sst"
w = RegridWorkflow(usecase, archive_root, target_root,
Period('1991-01-01', '1992-01-01'))
w.add_sensor('AVHRR10_G', (1986, 11, 1), (1991, 9, 16))
w.add_sensor('AVHRR11_G', (1988, 11, 1), (1994, 12, 31))
w.add_sensor('AVHRR12_G', (1991, 9, 1), (1998, 12, 14))
w.run(log_dir='.', simulation=True)
with open('regrid.status', 'r') as status:
self.assertEqual(
'25 created, 0 running, 0 backlog, 25 processed, 0 failed\n',
status.readline())
with open('regrid.report', 'r') as report:
self.assertEqual(25, len(report.readlines()))
os.remove('regrid.status')
os.remove('regrid.report')
def test_run_l2p_usecase(self):
usecase = 'l2p'
version = 'v2.1.8'
archive_root = '/group_workspaces/cems2/esacci_sst/output'
report_root = '/group_workspaces/cems2/esacci_sst/mms/svr'
w = SvrWorkflow(usecase, version, archive_root, report_root,
Period('1991-01-01', '1992-01-01'))
w.add_sensor('AVHRR10_G', (1986, 11, 17), (1991, 9, 16))
w.add_sensor('AVHRR11_G', (1988, 11, 8), (1994, 12, 31))
w.add_sensor('AVHRR12_G', (1991, 9, 16), (1998, 12, 14))
w.run(log_dir='.', simulation=True)
with open('l2p.status', 'r') as status:
self.assertEqual(
'736 created, 0 running, 0 backlog, 736 processed, 0 failed\n',
status.readline())
with open('l2p.report', 'r') as report:
self.assertEqual(736, len(report.readlines()))
os.remove('l2p.status')
os.remove('l2p.report')
def run(self,
hosts=list([('localhost', 120)]),
calls=list(),
log_dir='trace',
simulation=False):
"""
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
m = self.__get_monitor(hosts, calls, log_dir, simulation)
production_period = self.__get_effective_verification_period()
date = production_period.get_start_date()
while date < production_period.get_end_date():
chunk = Period(date, _next_year_start(date))
self._execute_verification(m, chunk)
date = _next_year_start(date)
self._execute_report_accumulation(m)
self._execute_report_plotting(m)
m.wait_for_completion_and_terminate()
from workflow import Period
from workflow import Workflow
usecase = 'mms7'
mmdtype = 'mmd7'
w = Workflow(usecase, Period('2012-07-02', '2015-01-01'))
w.add_primary_sensor('avhrr_f.m02', '2007-03-01', '2014-04-01')
w.add_secondary_sensor('amsr2', '2012-07-02', '2015-04-01')
w.set_samples_per_month(5000000)
w.run(mmdtype, hosts=[('localhost', 12)], without_arc=True)
from workflow import Period
from workflow import Workflow
usecase = 'mms6c'
mmdtype = 'mmd6'
w = Workflow(usecase, Period('2002-06-01', '2011-10-05'))
w.add_primary_sensor('amsre', '2002-06-01', '2011-10-05')
w.set_samples_per_month(3000000)
w.run(mmdtype,
hosts=[('localhost', 24)],
calls=[('sampling-start.sh', 1), ('coincidence-start.sh', 2),
('sub-start.sh', 2), ('mmd-start.sh', 2)],
with_history=True,
without_arc=True)
def test_get_sensor_name(self):
sensor = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
self.assertEqual('atsr.3', sensor.get_name())
def test_get_sensor_period(self):
sensor = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
self.assertEqual('2007-01-01',
sensor.get_period().get_start_date().isoformat())
self.assertEqual('2008-01-01',
sensor.get_period().get_end_date().isoformat())
def test_sensor_inequality(self):
sensor_1 = Sensor('atsr.3', Period((2007, 1, 1), (2008, 1, 1)))
sensor_2 = Sensor('atsr.2', Period((2007, 1, 1), (2008, 1, 1)))
sensor_3 = Sensor('atsr.3', Period((2008, 1, 1), (2009, 1, 1)))
self.assertTrue(sensor_1 != sensor_2)
self.assertTrue(sensor_1 != sensor_3)
def test_multi_period(self):
multi_period = MultiPeriod()
period_1 = Period('2007-01-01', '2007-02-01')
period_2 = Period('2007-02-01', '2007-03-01')
period_3 = Period('2006-12-01', '2007-01-01')
period_4 = Period('2006-11-01', '2007-01-01')
period_5 = Period('2007-02-01', '2007-04-01')
period_6 = Period('2007-05-01', '2007-06-01')
period_7 = Period('2007-04-01', '2007-05-01')
period_8 = Period('2007-11-01', '2007-12-01')
period_9 = Period('2007-09-01', '2007-10-01')
multi_period.add(period_1)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(period_1, periods[0])
multi_period.add(period_1)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(period_1, periods[0])
multi_period.add(period_2)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2007-01-01', '2007-03-01'), periods[0])
multi_period.add(period_3)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2006-12-01', '2007-03-01'), periods[0])
multi_period.add(period_4)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2006-11-01', '2007-03-01'), periods[0])
multi_period.add(period_5)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2006-11-01', '2007-04-01'), periods[0])
multi_period.add(period_6)
periods = multi_period.get_periods()
self.assertEqual(2, len(periods))
self.assertEqual(Period('2006-11-01', '2007-04-01'), periods[0])
self.assertEqual(period_6, periods[1])
multi_period.add(period_7)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2006-11-01', '2007-06-01'), periods[0])
multi_period.add(period_5)
periods = multi_period.get_periods()
self.assertEqual(1, len(periods))
self.assertEqual(Period('2006-11-01', '2007-06-01'), periods[0])
multi_period.add(period_8)
periods = multi_period.get_periods()
self.assertEqual(2, len(periods))
self.assertEqual(Period('2006-11-01', '2007-06-01'), periods[0])
self.assertEqual(period_8, periods[1])
multi_period.add(period_9)
periods = multi_period.get_periods()
self.assertEqual(3, len(periods))
self.assertEqual(Period('2006-11-01', '2007-06-01'), periods[0])
self.assertEqual(period_9, periods[1])
self.assertEqual(period_8, periods[2])
def test_period_inequality(self):
period_1 = Period((2007, 1, 1), (2008, 1, 1))
period_2 = Period((2007, 1, 1), (2009, 1, 1))
self.assertTrue(period_1 != period_2)
def test_get_period_intersection(self):
period_1 = Period('2007-01-01', '2008-01-01')
period_2 = Period('2007-07-01', '2008-07-01')
period_3 = Period('2007-10-01', '2007-11-01')
period_4 = Period('2001-07-01', '2002-07-01')
self.assertEqual(period_1, period_1.get_intersection(period_1))
self.assertEqual(period_2, period_2.get_intersection(period_2))
self.assertEqual(Period('2007-07-01', '2008-01-01'),
period_1.get_intersection(period_2))
self.assertEqual(Period('2007-07-01', '2008-01-01'),
period_2.get_intersection(period_1))
self.assertEqual(period_3, period_1.get_intersection(period_3))
self.assertTrue(period_1.get_intersection(period_4) is None)