def test_get_effective_production_period_one_sensor_with_workflow_period(
self):
w = Workflow('test', 2, 'config', Period((1994, 1, 1), (1994, 12, 31)))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
period = w._get_effective_production_period()
self.assertEqual(Period((1994, 1, 1), (1994, 12, 31)), period)
def test_add_get_primary_sensors_version(self):
w = Workflow('test', 5)
w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01',
'version_5')
self.assertEqual([
Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1)), 'version_5')
], w._get_primary_sensors())
def test_get_all_sensors_by_period(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
sensors = w._get_all_sensors_by_period()
""":type : list"""
self.assertEqual(3, len(sensors))
sensor_1 = sensors[0]
""":type : Sensor"""
sensor_2 = sensors[1]
""":type : Sensor"""
sensor_3 = sensors[2]
""":type : Sensor"""
self.assertEqual("avhrr.n10", sensor_1.get_name())
self.assertEqual(Period('1988-11-08', '1991-09-16'),
sensor_1.get_period())
self.assertEqual("avhrr.n11", sensor_2.get_name())
self.assertEqual(Period('1988-11-08', '1994-12-31'),
sensor_2.get_period())
self.assertEqual("avhrr.n12", sensor_3.get_name())
self.assertEqual(Period('1991-09-16', '1994-12-31'),
sensor_3.get_period())
def test_get_effective_production_period_many_sensors(self):
w = Workflow('test', 2)
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
period = w._get_effective_production_period()
self.assertEqual(Period((1988, 11, 8), (1991, 9, 16)), period)
def test_get_effective_production_period_many_sensors(self):
w = Workflow('test', 2)
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
period = w._get_effective_production_period()
self.assertEqual(Period((1988, 11, 8), (1991, 9, 16)), period)
def test_get_inp_preconditions_for_one_sensor(self):
w = Workflow('test', 8)
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
preconditions = list()
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(232, len(preconditions))
self.assertEqual('ingest-avhrr.n10-1986-321-1986-328', preconditions[0])
self.assertEqual('ingest-avhrr.n10-1989-048-1989-055', preconditions[108])
self.assertEqual('ingest-avhrr.n10-1991-252-1991-259', preconditions[231])
def test_add_get_primary_sensors_multiple(self):
w = Workflow('test', 5)
w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01')
w.add_primary_sensor('atsr-e1', '1991-06-01', '1995-01-01')
w.add_primary_sensor('atsr-en', '1996-01-01', '1998-01-01')
sensors = w._get_primary_sensors()
self.assertEqual([Sensor('atsr-en', Period((1996, 1, 1), (1998, 1, 1))),
Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1))),
Sensor('atsr-e1', Period((1991, 6, 1), (1995, 1, 1)))], sensors)
def test_ingest_avhrr_n07(self):
w = Workflow('test', 7,'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n07', '1981-09-01', '1985-01-30', 'v01.2')
w.run_ingestion(list([('localhost', 24)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual('203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline())
with open('test.report', 'r') as report:
self.assertEqual(203, len(report.readlines()))
def test_ingest_avhrr(self):
w = Workflow('test', 11, 'config/dir')
w.add_primary_sensor('avhrr-n12', '1995-06-01', '1996-06-05')
w.run_ingestion(list([('localhost', 5)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual('37 created, 0 running, 0 backlog, 37 processed, 0 failed\n', status.readline())
with open('test.report', 'r') as report:
self.assertEqual(37, len(report.readlines()))
def test_get_inp_preconditions_for_two_sensors(self):
w = Workflow('test', 14)
w.add_primary_sensor('avhrr.n11', (1988, 12, 18), (1990, 8, 15))
w.add_primary_sensor('avhrr.n12', (1985, 10, 16), (1988, 6, 23))
preconditions = list()
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(153, len(preconditions))
self.assertEqual('ingest-avhrr.n12-1985-289-1985-302', preconditions[0])
self.assertEqual('ingest-avhrr.n12-1986-210-1986-212', preconditions[27])
self.assertEqual('ingest-avhrr.n12-1987-166-1987-179', preconditions[58])
def test_add_get_primary_sensors_multiple(self):
w = Workflow('test', 5)
w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01')
w.add_primary_sensor('atsr-e1', '1991-06-01', '1995-01-01')
w.add_primary_sensor('atsr-en', '1996-01-01', '1998-01-01')
sensors = w._get_primary_sensors()
self.assertEqual([
Sensor('atsr-en', Period((1996, 1, 1), (1998, 1, 1))),
Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1))),
Sensor('atsr-e1', Period((1991, 6, 1), (1995, 1, 1)))
], sensors)
def test_get_data_period(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
data_period = w._get_data_period()
self.assertEqual(datetime.date(1988, 11, 8), data_period.get_start_date())
self.assertEqual(datetime.date(1994, 12, 31), data_period.get_end_date())
def test_get_inp_preconditions_for_one_month_minus_one_day(self):
w = Workflow('test', Period('1991-01-02', '1991-02-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = []
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(1, len(preconditions))
self.assertEqual('/inp/1991/01', preconditions[0])
def test_matchup_avhrr_n08_avhrr_n07(self):
w = Workflow('test', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n18', '2008-05-01', '2008-05-31', 'v01.2')
w.add_secondary_sensor('avhrr-n17', '2008-05-01', '2008-05-31', 'v01.2')
w.set_usecase_config('usecase-02.xml')
w.run_matchup(list([('localhost', 4)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual('5 created, 0 running, 0 backlog, 5 processed, 0 failed\n', status.readline())
with open('test.report', 'r') as report:
self.assertEqual(5, len(report.readlines()))
def test_get_sensor_pairs(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
sensor_pairs = w._get_sensor_pairs()
self.assertEqual(2, len(sensor_pairs))
self.assertEqual('avhrr.n12', sensor_pairs[0].get_primary_name())
self.assertEqual('avhrr.n11', sensor_pairs[1].get_primary_name())
def test_get_inp_preconditions_for_one_sensor(self):
w = Workflow('test', 8)
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
preconditions = list()
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(232, len(preconditions))
self.assertEqual('ingest-avhrr.n10-1986-321-1986-328',
preconditions[0])
self.assertEqual('ingest-avhrr.n10-1989-048-1989-055',
preconditions[108])
self.assertEqual('ingest-avhrr.n10-1991-252-1991-259',
preconditions[231])
def test_ingest_avhrr(self):
w = Workflow('test', 11, 'config/dir')
w.add_primary_sensor('avhrr-n12', '1995-06-01', '1996-06-05')
w.run_ingestion(list([('localhost', 5)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual(
'37 created, 0 running, 0 backlog, 37 processed, 0 failed\n',
status.readline())
with open('test.report', 'r') as report:
self.assertEqual(37, len(report.readlines()))
def test_get_sensor_pairs(self):
w = Workflow('test', 4)
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
sensor_pairs = w._get_sensor_pairs()
self.assertEqual(2, len(sensor_pairs))
self.assertEqual('avhrr.n12', sensor_pairs[0].get_primary_name())
self.assertEqual('avhrr.n11', sensor_pairs[1].get_primary_name())
def test_get_inp_preconditions_for_one_month_minus_one_day(self):
w = Workflow('test', Period('1991-01-02', '1991-02-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = []
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(1, len(preconditions))
self.assertEqual('/inp/1991/01', preconditions[0])
def test_get_data_period(self):
w = Workflow('test', 2)
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
data_period = w._get_data_period()
self.assertEqual(datetime.date(1988, 11, 8),
data_period.get_start_date())
self.assertEqual(datetime.date(1994, 12, 31),
data_period.get_end_date())
def test_get_inp_preconditions_for_two_sensors(self):
w = Workflow('test', 14)
w.add_primary_sensor('avhrr.n11', (1988, 12, 18), (1990, 8, 15))
w.add_primary_sensor('avhrr.n12', (1985, 10, 16), (1988, 6, 23))
preconditions = list()
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(153, len(preconditions))
self.assertEqual('ingest-avhrr.n12-1985-289-1985-302',
preconditions[0])
self.assertEqual('ingest-avhrr.n12-1986-210-1986-212',
preconditions[27])
self.assertEqual('ingest-avhrr.n12-1987-166-1987-179',
preconditions[58])
def test_ingest_avhrr_n07(self):
w = Workflow('test', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n07', '1981-09-01', '1985-01-30', 'v01.2')
w.run_ingestion(list([('localhost', 24)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual(
'203 created, 0 running, 0 backlog, 203 processed, 0 failed\n',
status.readline())
with open('test.report', 'r') as report:
self.assertEqual(203, len(report.readlines()))
def test_add_obs_preconditions_for_one_year_and_one_day(self):
w = Workflow('test', Period('1991-01-01', '1992-01-02'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_obs_preconditions(preconditions)
self.assertEqual(2, len(preconditions))
self.assertEqual('/obs/1990/12', preconditions[0])
self.assertEqual('/obs/1992/02', preconditions[1])
def test_add_obs_preconditions_for_all_years(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_obs_preconditions(preconditions)
self.assertEqual(2, len(preconditions))
self.assertEqual('/obs/1988/10', preconditions[0])
self.assertEqual('/obs/1995/01', preconditions[1])
def test_add_obs_preconditions_for_all_years(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_obs_preconditions(preconditions)
self.assertEqual(2, len(preconditions))
self.assertEqual('/obs/1988/10', preconditions[0])
self.assertEqual('/obs/1995/01', preconditions[1])
def test_get_inp_preconditions_for_all_years(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = []
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(74, len(preconditions))
self.assertEqual('/inp/1988/11', preconditions[0])
self.assertEqual('/inp/1994/12', preconditions[73])
def test_add_obs_preconditions_for_one_year_and_one_day(self):
w = Workflow('test', Period('1991-01-01', '1992-01-02'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_obs_preconditions(preconditions)
self.assertEqual(2, len(preconditions))
self.assertEqual('/obs/1990/12', preconditions[0])
self.assertEqual('/obs/1992/02', preconditions[1])
def test_get_inp_preconditions_for_all_years(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = []
preconditions = w._add_inp_preconditions(preconditions)
self.assertEqual(74, len(preconditions))
self.assertEqual('/inp/1988/11', preconditions[0])
self.assertEqual('/inp/1994/12', preconditions[73])
def test_add_smp_preconditions_for_one_year(self):
w = Workflow('test', Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_smp_preconditions(preconditions)
self.assertEqual(4, len(preconditions))
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1991/08', preconditions[0])
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1992/01', preconditions[1])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1990/12', preconditions[2])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1991/10', preconditions[3])
def test_run_single_sensor_usecase(self):
usecase = 'mms3'
mmdtype = 'mmd3'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', with_history=True, simulation=True)
with open('mms3.status', 'r') as status:
self.assertEqual('337 created, 0 running, 0 backlog, 337 processed, 0 failed\n', status.readline())
with open('mms3.report', 'r') as report:
self.assertEqual(337, len(report.readlines()))
os.remove('mms3.status')
os.remove('mms3.report')
def test_add_smp_preconditions_for_one_year(self):
w = Workflow('test', Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
preconditions = list()
preconditions = w._add_smp_preconditions(preconditions)
self.assertEqual(4, len(preconditions))
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1991/08', preconditions[0])
self.assertEqual('/smp/avhrr.n12,avhrr.n11/1992/01', preconditions[1])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1990/12', preconditions[2])
self.assertEqual('/smp/avhrr.n11,avhrr.n10/1991/10', preconditions[3])
def test_matchup_avhrr_n08_avhrr_n07(self):
w = Workflow('test', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n18', '2008-05-01', '2008-05-31', 'v01.2')
w.add_secondary_sensor('avhrr-n17', '2008-05-01', '2008-05-31',
'v01.2')
w.set_usecase_config('usecase-02.xml')
w.run_matchup(list([('localhost', 4)]), True, self.logdir)
with open('test.status', 'r') as status:
self.assertEqual(
'5 created, 0 running, 0 backlog, 5 processed, 0 failed\n',
status.readline())
with open('test.report', 'r') as report:
self.assertEqual(5, len(report.readlines()))
def test_run_single_sensor_usecase(self):
usecase = 'mms3'
mmdtype = 'mmd3'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', with_history=True, simulation=True)
with open('mms3.status', 'r') as status:
self.assertEqual(
'337 created, 0 running, 0 backlog, 337 processed, 0 failed\n',
status.readline())
with open('mms3.report', 'r') as report:
self.assertEqual(337, len(report.readlines()))
os.remove('mms3.status')
os.remove('mms3.report')
def test_run_dual_sensor_usecase_without_arc(self):
usecase = 'mms1'
mmdtype = 'mmd1'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', simulation=True, without_arc=True)
with open('mms1.status', 'r') as status:
self.assertEqual('203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline())
with open('mms1.report', 'r') as report:
self.assertEqual(203, len(report.readlines()))
os.remove('mms1.status')
os.remove('mms1.report')
def test_get_primary_sensors_by_period(self):
w = Workflow('test')
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
sensors = w._get_primary_sensors_by_period()
""":type : list"""
self.assertEqual(2, len(sensors))
sensor_1 = sensors[0]
""":type : Sensor"""
sensor_2 = sensors[1]
""":type : Sensor"""
self.assertEqual("avhrr.n11", sensor_1.get_name())
self.assertEqual(Period('1988-11-08', '1991-09-16'), sensor_1.get_period())
self.assertEqual("avhrr.n12", sensor_2.get_name())
self.assertEqual(Period('1991-09-16', '1994-12-31'), sensor_2.get_period())
def test_run_dual_sensor_usecase_without_arc(self):
usecase = 'mms1'
mmdtype = 'mmd1'
w = Workflow(usecase, Period('1991-01-01', '1992-01-01'))
w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16))
w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31))
w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14))
w.run(mmdtype, log_dir='.', simulation=True, without_arc=True)
with open('mms1.status', 'r') as status:
self.assertEqual(
'203 created, 0 running, 0 backlog, 203 processed, 0 failed\n',
status.readline())
with open('mms1.report', 'r') as report:
self.assertEqual(203, len(report.readlines()))
os.remove('mms1.status')
os.remove('mms1.report')
def test_add_primary_sensors_to_workflow(self):
w = Workflow('test')
w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1))
self.assertEqual(1, len(w._get_primary_sensors()))
try:
w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1))
self.fail()
except exceptions.ValueError:
pass
try:
w.add_primary_sensor('atsr.3', (2007, 7, 1), (2008, 1, 1))
self.fail()
except exceptions.ValueError:
pass
w.add_primary_sensor('atsr.2', (2007, 1, 1), (2008, 1, 1))
self.assertEqual(2, len(w._get_primary_sensors()))
w.add_primary_sensor('atsr.3', (2008, 1, 1), (2009, 1, 1))
self.assertEqual(3, len(w._get_primary_sensors()))
def test_add_primary_sensors_to_workflow(self):
w = Workflow('test')
w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1))
self.assertEqual(1, len(w._get_primary_sensors()))
try:
w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1))
self.fail()
except exceptions.ValueError:
pass
try:
w.add_primary_sensor('atsr.3', (2007, 7, 1), (2008, 1, 1))
self.fail()
except exceptions.ValueError:
pass
w.add_primary_sensor('atsr.2', (2007, 1, 1), (2008, 1, 1))
self.assertEqual(2, len(w._get_primary_sensors()))
w.add_primary_sensor('atsr.3', (2008, 1, 1), (2009, 1, 1))
self.assertEqual(3, len(w._get_primary_sensors()))
from workflow import Period
from workflow import Workflow
usecase = 'mms11b'
mmdtype = 'mmd11'
w = Workflow(usecase)
w.add_primary_sensor('avhrr.n07', '1981-09-01', '1985-02-01')
w.add_primary_sensor('avhrr.n08', '1983-05-04', '1985-10-03')
w.add_primary_sensor('avhrr.n09', '1985-02-25', '1988-11-07')
w.add_primary_sensor('avhrr.n10', '1986-11-17', '1991-09-17')
w.add_primary_sensor('avhrr.n11', '1988-11-08', '1994-09-14')
#w.add_primary_sensor('avhrr.n12', '1991-09-16', '1998-12-15')
#w.add_primary_sensor('avhrr.n14', '1995-01-01', '2000-01-01')
#w.add_primary_sensor('avhrr.n15', '1998-10-26', '2003-04-09')
#w.add_primary_sensor('avhrr.n15', '2003-12-21', '2011-01-01')
#w.add_primary_sensor('avhrr.n16', '2001-01-01', '2003-04-09')
#w.add_primary_sensor('avhrr.n16', '2003-12-21', '2011-01-01')
#w.add_primary_sensor('avhrr.n17', '2002-07-10', '2003-04-09')
#w.add_primary_sensor('avhrr.n17', '2003-12-21', '2011-01-01')
#w.add_primary_sensor('avhrr.n18', '2005-06-05', '2014-01-01')
#w.add_primary_sensor('avhrr.n19', '2009-02-06', '2014-01-01')
#w.add_primary_sensor('avhrr.m02', '2006-11-21', '2014-01-01')
w.add_secondary_sensor('avhrr.n07', '1981-09-01', '1985-02-01')
w.add_secondary_sensor('avhrr.n08', '1983-05-04', '1985-10-03')
w.add_secondary_sensor('avhrr.n09', '1985-02-25', '1988-11-07')
w.add_secondary_sensor('avhrr.n10', '1986-11-17', '1991-09-17')
w.add_secondary_sensor('avhrr.n11', '1988-11-08', '1994-09-14')
#w.add_secondary_sensor('avhrr.n12', '1991-09-16', '1998-12-15')
#w.add_secondary_sensor('avhrr.n14', '1995-01-01', '2000-01-01')
#w.add_secondary_sensor('avhrr.n15', '1998-10-26', '2003-04-09')
from workflow import Workflow
usecase = "mms14"
mmdtype = "mmd14"
w = Workflow(usecase)
# the following lines define the full MMD tb 2016-08-01
# w.add_primary_sensor('atsr.3', '2002-05-20', '2012-04-09')
# w.add_primary_sensor('amsre', '2002-06-01', '2011-10-05')
# the following lines define the test MMD tb 2016-08-01
w.add_primary_sensor("atsr.3", "2008-05-01", "2008-06-01")
w.add_secondary_sensor("amsre", "2008-05-01", "2008-06-01")
w.set_samples_per_month(100000)
w.run(mmdtype, hosts=[("localhost", 8)])
from workflow import Workflow
w = Workflow('usecase02_avhrr_n18_n16', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n18', '2005-05-20', '2010-12-31', 'v01.2')
w.add_secondary_sensor('avhrr-n16', '2005-05-20', '2010-12-31', 'v01.2')
w.set_usecase_config('usecase-02.xml')
w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase05_hirs_n17_n16', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('hirs-n17', '2002-07-10', '2013-04-09', '1.0')
w.add_secondary_sensor('hirs-n16', '2002-07-10', '2013-04-09', '1.0')
w.set_usecase_config('usecase-05.xml')
w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase01_aatsr_en_avhrr_m02', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('aatsr-en', '2006-10-30', '2012-04-08', 'v3')
w.add_secondary_sensor('avhrr-m02', '2006-10-30', '2012-04-08', 'v01.2')
w.set_usecase_config('usecase-01.xml')
w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow
# Parameters
# 1 - usecase name
# 2 - number of days per time slot
# 3 - configuration directory
w = Workflow('ingest_aatsr_en', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('aatsr-en', '2002-05-20', '2012-04-08', 'v3')
# Parameters
# 1 - host (usually localhost), number of tasks to submit to scheduler at once
# 2 - number of parallel executed tasks
w.run_ingestion(hosts=[('localhost', 10)], num_parallel_tasks=24)
from workflow import Workflow
w = Workflow('ingest_avhrr_n09', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n09', '1985-02-27', '1988-11-07', 'v01.2')
w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow
usecase = 'mms12'
mmdtype = 'mmd12'
w = Workflow(usecase)
w.add_primary_sensor('avhrr_f.m01', '2012-12-13', '2016-01-04')
w.add_primary_sensor('avhrr_f.m02', '2007-03-01', '2016-01-04')
w.set_samples_per_month(0)
w.run(mmdtype,
hosts=[('localhost', 8)],
calls=[('sampling-start.sh', 1), ('coincidence-start.sh', 1)],
with_history=True)
from workflow import Workflow
w = Workflow('usecase05_hirs_n18_n16', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('hirs-n18', '2006-01-01', '2014-06-05', '1.0')
w.add_secondary_sensor('hirs-n16', '2006-01-01', '2014-06-05', '1.0')
w.set_usecase_config('usecase-05.xml')
w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase17_mhs_n18_amsub_n17', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('mhs-n18', '2005-05-25', '2013-04-10', 'v1.0')
w.add_secondary_sensor('amsub-n17', '2005-05-25', '2013-04-10', 'v1.0')
w.set_usecase_config('usecase-17.xml')
w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow
w = Workflow('ingest_avhrr_n19', 7,
'/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n19', '2009-02-07', '2015-12-31', 'v01.2')
w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase02_avhrr_n16_n15', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n16', '2001-01-01', '2010-12-31', 'v01.2')
w.add_secondary_sensor('avhrr-n15', '2001-01-01', '2010-12-31', 'v01.2')
w.set_usecase_config('usecase-02.xml')
w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow
usecase = 'mms3'
mmdtype = 'mmd3'
w = Workflow(usecase)
w.add_primary_sensor('atsr.1', '1991-08-01', '1996-09-01')
w.add_primary_sensor('atsr.1', '1996-10-01', '1996-11-01')
w.add_primary_sensor('atsr.1', '1996-12-30', '1997-02-01')
w.add_primary_sensor('atsr.1', '1997-03-01', '1997-04-01')
w.add_primary_sensor('atsr.1', '1997-05-01', '1997-06-01')
w.add_primary_sensor('atsr.1', '1997-07-01', '1997-09-01')
w.add_primary_sensor('atsr.1', '1997-10-01', '1997-11-01')
w.add_primary_sensor('atsr.1', '1997-12-01', '1997-12-18')
w.add_primary_sensor('atsr.2', '1995-06-01', '1996-01-01')
w.add_primary_sensor('atsr.2', '1996-07-01', '2003-06-23')
w.add_primary_sensor('atsr.3', '2002-05-20', '2012-04-09')
w.set_samples_per_month(0)
w.run(mmdtype, hosts=[('localhost', 24)], calls=[('sampling-start', 1)], with_history=True)
from workflow import Period
from workflow import Workflow
usecase = 'mms7'
mmdtype = 'mmd7'
w = Workflow(usecase, Period('2012-07-02', '2015-01-01'))
w.add_primary_sensor('avhrr_f.m02', '2007-03-01', '2014-04-01')
w.add_secondary_sensor('amsr2', '2012-07-02', '2015-04-01')
w.set_samples_per_month(5000000)
w.run(mmdtype, hosts=[('localhost', 12)], without_arc=True)
from workflow import Workflow
w = Workflow('ingest_atsr_e1', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('atsr-e1', '1991-08-01', '1997-12-17', 'v3')
w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase06c_sst_mbt_sst_amsre_aq', 7, '/group_workspaces/cems2/esacci_sst/mms_new/config')
w.add_primary_sensor('mbt-sst', '2002-06-01', '2011-10-04', 'v03.3')
w.add_secondary_sensor('amsre-aq', '2002-06-01', '2011-10-04', 'v12')
w.set_usecase_config('usecase-06s-sst.xml')
w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow
w = Workflow('usecase01_atsr_e2_avhrr_m02', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('atsr-e2', '2006-10-30', '2008-01-31', 'v3')
w.add_secondary_sensor('avhrr-m02', '2006-10-30', '2008-01-31', 'v01.2')
w.set_usecase_config('usecase-01.xml')
w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow
w = Workflow('usecase02_avhrr_n19_iasi_mb', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config')
w.add_primary_sensor('avhrr-n19', '2013-02-20', '2015-12-31', 'v01.2')
w.add_secondary_sensor('iasi-mb', '2013-02-20', '2015-12-31', 'latest')
w.set_usecase_config('usecase-03.xml')
w.run_matchup(hosts=[('localhost', 72)])
