def test_get_effective_production_period_one_sensor_with_workflow_period( self): w = Workflow('test', 2, 'config', Period((1994, 1, 1), (1994, 12, 31))) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) period = w._get_effective_production_period() self.assertEqual(Period((1994, 1, 1), (1994, 12, 31)), period)
def test_add_get_primary_sensors_version(self): w = Workflow('test', 5) w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01', 'version_5') self.assertEqual([ Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1)), 'version_5') ], w._get_primary_sensors())
def test_get_all_sensors_by_period(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) sensors = w._get_all_sensors_by_period() """:type : list""" self.assertEqual(3, len(sensors)) sensor_1 = sensors[0] """:type : Sensor""" sensor_2 = sensors[1] """:type : Sensor""" sensor_3 = sensors[2] """:type : Sensor""" self.assertEqual("avhrr.n10", sensor_1.get_name()) self.assertEqual(Period('1988-11-08', '1991-09-16'), sensor_1.get_period()) self.assertEqual("avhrr.n11", sensor_2.get_name()) self.assertEqual(Period('1988-11-08', '1994-12-31'), sensor_2.get_period()) self.assertEqual("avhrr.n12", sensor_3.get_name()) self.assertEqual(Period('1991-09-16', '1994-12-31'), sensor_3.get_period())
def test_get_effective_production_period_many_sensors(self): w = Workflow('test', 2) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) period = w._get_effective_production_period() self.assertEqual(Period((1988, 11, 8), (1991, 9, 16)), period)
def test_get_inp_preconditions_for_one_sensor(self): w = Workflow('test', 8) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) preconditions = list() preconditions = w._add_inp_preconditions(preconditions) self.assertEqual(232, len(preconditions)) self.assertEqual('ingest-avhrr.n10-1986-321-1986-328', preconditions[0]) self.assertEqual('ingest-avhrr.n10-1989-048-1989-055', preconditions[108]) self.assertEqual('ingest-avhrr.n10-1991-252-1991-259', preconditions[231])
def test_add_get_primary_sensors_multiple(self): w = Workflow('test', 5) w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01') w.add_primary_sensor('atsr-e1', '1991-06-01', '1995-01-01') w.add_primary_sensor('atsr-en', '1996-01-01', '1998-01-01') sensors = w._get_primary_sensors() self.assertEqual([Sensor('atsr-en', Period((1996, 1, 1), (1998, 1, 1))), Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1))), Sensor('atsr-e1', Period((1991, 6, 1), (1995, 1, 1)))], sensors)
def test_ingest_avhrr_n07(self): w = Workflow('test', 7,'/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n07', '1981-09-01', '1985-01-30', 'v01.2') w.run_ingestion(list([('localhost', 24)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual('203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(203, len(report.readlines()))
def test_ingest_avhrr(self): w = Workflow('test', 11, 'config/dir') w.add_primary_sensor('avhrr-n12', '1995-06-01', '1996-06-05') w.run_ingestion(list([('localhost', 5)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual('37 created, 0 running, 0 backlog, 37 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(37, len(report.readlines()))
def test_get_inp_preconditions_for_two_sensors(self): w = Workflow('test', 14) w.add_primary_sensor('avhrr.n11', (1988, 12, 18), (1990, 8, 15)) w.add_primary_sensor('avhrr.n12', (1985, 10, 16), (1988, 6, 23)) preconditions = list() preconditions = w._add_inp_preconditions(preconditions) self.assertEqual(153, len(preconditions)) self.assertEqual('ingest-avhrr.n12-1985-289-1985-302', preconditions[0]) self.assertEqual('ingest-avhrr.n12-1986-210-1986-212', preconditions[27]) self.assertEqual('ingest-avhrr.n12-1987-166-1987-179', preconditions[58])
def test_add_get_primary_sensors_multiple(self): w = Workflow('test', 5) w.add_primary_sensor('atsr-e2', '1995-06-01', '1996-01-01') w.add_primary_sensor('atsr-e1', '1991-06-01', '1995-01-01') w.add_primary_sensor('atsr-en', '1996-01-01', '1998-01-01') sensors = w._get_primary_sensors() self.assertEqual([ Sensor('atsr-en', Period((1996, 1, 1), (1998, 1, 1))), Sensor('atsr-e2', Period((1995, 6, 1), (1996, 1, 1))), Sensor('atsr-e1', Period((1991, 6, 1), (1995, 1, 1))) ], sensors)
def test_get_data_period(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) data_period = w._get_data_period() self.assertEqual(datetime.date(1988, 11, 8), data_period.get_start_date()) self.assertEqual(datetime.date(1994, 12, 31), data_period.get_end_date())
def test_get_inp_preconditions_for_one_month_minus_one_day(self): w = Workflow('test', Period('1991-01-02', '1991-02-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) preconditions = [] preconditions = w._add_inp_preconditions(preconditions) self.assertEqual(1, len(preconditions)) self.assertEqual('/inp/1991/01', preconditions[0])
def test_matchup_avhrr_n08_avhrr_n07(self): w = Workflow('test', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n18', '2008-05-01', '2008-05-31', 'v01.2') w.add_secondary_sensor('avhrr-n17', '2008-05-01', '2008-05-31', 'v01.2') w.set_usecase_config('usecase-02.xml') w.run_matchup(list([('localhost', 4)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual('5 created, 0 running, 0 backlog, 5 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(5, len(report.readlines()))
def test_get_sensor_pairs(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) sensor_pairs = w._get_sensor_pairs() self.assertEqual(2, len(sensor_pairs)) self.assertEqual('avhrr.n12', sensor_pairs[0].get_primary_name()) self.assertEqual('avhrr.n11', sensor_pairs[1].get_primary_name())
def test_ingest_avhrr(self): w = Workflow('test', 11, 'config/dir') w.add_primary_sensor('avhrr-n12', '1995-06-01', '1996-06-05') w.run_ingestion(list([('localhost', 5)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual( '37 created, 0 running, 0 backlog, 37 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(37, len(report.readlines()))
def test_get_sensor_pairs(self): w = Workflow('test', 4) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) sensor_pairs = w._get_sensor_pairs() self.assertEqual(2, len(sensor_pairs)) self.assertEqual('avhrr.n12', sensor_pairs[0].get_primary_name()) self.assertEqual('avhrr.n11', sensor_pairs[1].get_primary_name())
def test_get_data_period(self): w = Workflow('test', 2) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) data_period = w._get_data_period() self.assertEqual(datetime.date(1988, 11, 8), data_period.get_start_date()) self.assertEqual(datetime.date(1994, 12, 31), data_period.get_end_date())
def test_ingest_avhrr_n07(self): w = Workflow('test', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n07', '1981-09-01', '1985-01-30', 'v01.2') w.run_ingestion(list([('localhost', 24)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual( '203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(203, len(report.readlines()))
def test_add_obs_preconditions_for_one_year_and_one_day(self): w = Workflow('test', Period('1991-01-01', '1992-01-02')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) preconditions = list() preconditions = w._add_obs_preconditions(preconditions) self.assertEqual(2, len(preconditions)) self.assertEqual('/obs/1990/12', preconditions[0]) self.assertEqual('/obs/1992/02', preconditions[1])
def test_add_obs_preconditions_for_all_years(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) preconditions = list() preconditions = w._add_obs_preconditions(preconditions) self.assertEqual(2, len(preconditions)) self.assertEqual('/obs/1988/10', preconditions[0]) self.assertEqual('/obs/1995/01', preconditions[1])
def test_get_inp_preconditions_for_all_years(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) preconditions = [] preconditions = w._add_inp_preconditions(preconditions) self.assertEqual(74, len(preconditions)) self.assertEqual('/inp/1988/11', preconditions[0]) self.assertEqual('/inp/1994/12', preconditions[73])
def test_add_smp_preconditions_for_one_year(self): w = Workflow('test', Period('1991-01-01', '1992-01-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) preconditions = list() preconditions = w._add_smp_preconditions(preconditions) self.assertEqual(4, len(preconditions)) self.assertEqual('/smp/avhrr.n12,avhrr.n11/1991/08', preconditions[0]) self.assertEqual('/smp/avhrr.n12,avhrr.n11/1992/01', preconditions[1]) self.assertEqual('/smp/avhrr.n11,avhrr.n10/1990/12', preconditions[2]) self.assertEqual('/smp/avhrr.n11,avhrr.n10/1991/10', preconditions[3])
def test_run_single_sensor_usecase(self): usecase = 'mms3' mmdtype = 'mmd3' w = Workflow(usecase, Period('1991-01-01', '1992-01-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.run(mmdtype, log_dir='.', with_history=True, simulation=True) with open('mms3.status', 'r') as status: self.assertEqual('337 created, 0 running, 0 backlog, 337 processed, 0 failed\n', status.readline()) with open('mms3.report', 'r') as report: self.assertEqual(337, len(report.readlines())) os.remove('mms3.status') os.remove('mms3.report')
def test_matchup_avhrr_n08_avhrr_n07(self): w = Workflow('test', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n18', '2008-05-01', '2008-05-31', 'v01.2') w.add_secondary_sensor('avhrr-n17', '2008-05-01', '2008-05-31', 'v01.2') w.set_usecase_config('usecase-02.xml') w.run_matchup(list([('localhost', 4)]), True, self.logdir) with open('test.status', 'r') as status: self.assertEqual( '5 created, 0 running, 0 backlog, 5 processed, 0 failed\n', status.readline()) with open('test.report', 'r') as report: self.assertEqual(5, len(report.readlines()))
def test_run_single_sensor_usecase(self): usecase = 'mms3' mmdtype = 'mmd3' w = Workflow(usecase, Period('1991-01-01', '1992-01-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.run(mmdtype, log_dir='.', with_history=True, simulation=True) with open('mms3.status', 'r') as status: self.assertEqual( '337 created, 0 running, 0 backlog, 337 processed, 0 failed\n', status.readline()) with open('mms3.report', 'r') as report: self.assertEqual(337, len(report.readlines())) os.remove('mms3.status') os.remove('mms3.report')
def test_run_dual_sensor_usecase_without_arc(self): usecase = 'mms1' mmdtype = 'mmd1' w = Workflow(usecase, Period('1991-01-01', '1992-01-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.run(mmdtype, log_dir='.', simulation=True, without_arc=True) with open('mms1.status', 'r') as status: self.assertEqual('203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline()) with open('mms1.report', 'r') as report: self.assertEqual(203, len(report.readlines())) os.remove('mms1.status') os.remove('mms1.report')
def test_get_primary_sensors_by_period(self): w = Workflow('test') w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) sensors = w._get_primary_sensors_by_period() """:type : list""" self.assertEqual(2, len(sensors)) sensor_1 = sensors[0] """:type : Sensor""" sensor_2 = sensors[1] """:type : Sensor""" self.assertEqual("avhrr.n11", sensor_1.get_name()) self.assertEqual(Period('1988-11-08', '1991-09-16'), sensor_1.get_period()) self.assertEqual("avhrr.n12", sensor_2.get_name()) self.assertEqual(Period('1991-09-16', '1994-12-31'), sensor_2.get_period())
def test_run_dual_sensor_usecase_without_arc(self): usecase = 'mms1' mmdtype = 'mmd1' w = Workflow(usecase, Period('1991-01-01', '1992-01-01')) w.add_primary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_primary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_primary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.add_secondary_sensor('avhrr.n10', (1986, 11, 17), (1991, 9, 16)) w.add_secondary_sensor('avhrr.n11', (1988, 11, 8), (1994, 12, 31)) w.add_secondary_sensor('avhrr.n12', (1991, 9, 16), (1998, 12, 14)) w.run(mmdtype, log_dir='.', simulation=True, without_arc=True) with open('mms1.status', 'r') as status: self.assertEqual( '203 created, 0 running, 0 backlog, 203 processed, 0 failed\n', status.readline()) with open('mms1.report', 'r') as report: self.assertEqual(203, len(report.readlines())) os.remove('mms1.status') os.remove('mms1.report')
def test_add_primary_sensors_to_workflow(self): w = Workflow('test') w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1)) self.assertEqual(1, len(w._get_primary_sensors())) try: w.add_primary_sensor('atsr.3', (2007, 1, 1), (2008, 1, 1)) self.fail() except exceptions.ValueError: pass try: w.add_primary_sensor('atsr.3', (2007, 7, 1), (2008, 1, 1)) self.fail() except exceptions.ValueError: pass w.add_primary_sensor('atsr.2', (2007, 1, 1), (2008, 1, 1)) self.assertEqual(2, len(w._get_primary_sensors())) w.add_primary_sensor('atsr.3', (2008, 1, 1), (2009, 1, 1)) self.assertEqual(3, len(w._get_primary_sensors()))
from workflow import Period from workflow import Workflow usecase = 'mms11b' mmdtype = 'mmd11' w = Workflow(usecase) w.add_primary_sensor('avhrr.n07', '1981-09-01', '1985-02-01') w.add_primary_sensor('avhrr.n08', '1983-05-04', '1985-10-03') w.add_primary_sensor('avhrr.n09', '1985-02-25', '1988-11-07') w.add_primary_sensor('avhrr.n10', '1986-11-17', '1991-09-17') w.add_primary_sensor('avhrr.n11', '1988-11-08', '1994-09-14') #w.add_primary_sensor('avhrr.n12', '1991-09-16', '1998-12-15') #w.add_primary_sensor('avhrr.n14', '1995-01-01', '2000-01-01') #w.add_primary_sensor('avhrr.n15', '1998-10-26', '2003-04-09') #w.add_primary_sensor('avhrr.n15', '2003-12-21', '2011-01-01') #w.add_primary_sensor('avhrr.n16', '2001-01-01', '2003-04-09') #w.add_primary_sensor('avhrr.n16', '2003-12-21', '2011-01-01') #w.add_primary_sensor('avhrr.n17', '2002-07-10', '2003-04-09') #w.add_primary_sensor('avhrr.n17', '2003-12-21', '2011-01-01') #w.add_primary_sensor('avhrr.n18', '2005-06-05', '2014-01-01') #w.add_primary_sensor('avhrr.n19', '2009-02-06', '2014-01-01') #w.add_primary_sensor('avhrr.m02', '2006-11-21', '2014-01-01') w.add_secondary_sensor('avhrr.n07', '1981-09-01', '1985-02-01') w.add_secondary_sensor('avhrr.n08', '1983-05-04', '1985-10-03') w.add_secondary_sensor('avhrr.n09', '1985-02-25', '1988-11-07') w.add_secondary_sensor('avhrr.n10', '1986-11-17', '1991-09-17') w.add_secondary_sensor('avhrr.n11', '1988-11-08', '1994-09-14') #w.add_secondary_sensor('avhrr.n12', '1991-09-16', '1998-12-15') #w.add_secondary_sensor('avhrr.n14', '1995-01-01', '2000-01-01') #w.add_secondary_sensor('avhrr.n15', '1998-10-26', '2003-04-09')
from workflow import Workflow usecase = "mms14" mmdtype = "mmd14" w = Workflow(usecase) # the following lines define the full MMD tb 2016-08-01 # w.add_primary_sensor('atsr.3', '2002-05-20', '2012-04-09') # w.add_primary_sensor('amsre', '2002-06-01', '2011-10-05') # the following lines define the test MMD tb 2016-08-01 w.add_primary_sensor("atsr.3", "2008-05-01", "2008-06-01") w.add_secondary_sensor("amsre", "2008-05-01", "2008-06-01") w.set_samples_per_month(100000) w.run(mmdtype, hosts=[("localhost", 8)])
from workflow import Workflow w = Workflow('usecase02_avhrr_n18_n16', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n18', '2005-05-20', '2010-12-31', 'v01.2') w.add_secondary_sensor('avhrr-n16', '2005-05-20', '2010-12-31', 'v01.2') w.set_usecase_config('usecase-02.xml') w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase05_hirs_n17_n16', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('hirs-n17', '2002-07-10', '2013-04-09', '1.0') w.add_secondary_sensor('hirs-n16', '2002-07-10', '2013-04-09', '1.0') w.set_usecase_config('usecase-05.xml') w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase01_aatsr_en_avhrr_m02', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('aatsr-en', '2006-10-30', '2012-04-08', 'v3') w.add_secondary_sensor('avhrr-m02', '2006-10-30', '2012-04-08', 'v01.2') w.set_usecase_config('usecase-01.xml') w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow # Parameters # 1 - usecase name # 2 - number of days per time slot # 3 - configuration directory w = Workflow('ingest_aatsr_en', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('aatsr-en', '2002-05-20', '2012-04-08', 'v3') # Parameters # 1 - host (usually localhost), number of tasks to submit to scheduler at once # 2 - number of parallel executed tasks w.run_ingestion(hosts=[('localhost', 10)], num_parallel_tasks=24)
from workflow import Workflow w = Workflow('ingest_avhrr_n09', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n09', '1985-02-27', '1988-11-07', 'v01.2') w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow usecase = 'mms12' mmdtype = 'mmd12' w = Workflow(usecase) w.add_primary_sensor('avhrr_f.m01', '2012-12-13', '2016-01-04') w.add_primary_sensor('avhrr_f.m02', '2007-03-01', '2016-01-04') w.set_samples_per_month(0) w.run(mmdtype, hosts=[('localhost', 8)], calls=[('sampling-start.sh', 1), ('coincidence-start.sh', 1)], with_history=True)
from workflow import Workflow w = Workflow('usecase05_hirs_n18_n16', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('hirs-n18', '2006-01-01', '2014-06-05', '1.0') w.add_secondary_sensor('hirs-n16', '2006-01-01', '2014-06-05', '1.0') w.set_usecase_config('usecase-05.xml') w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase17_mhs_n18_amsub_n17', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('mhs-n18', '2005-05-25', '2013-04-10', 'v1.0') w.add_secondary_sensor('amsub-n17', '2005-05-25', '2013-04-10', 'v1.0') w.set_usecase_config('usecase-17.xml') w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow w = Workflow('ingest_avhrr_n19', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n19', '2009-02-07', '2015-12-31', 'v01.2') w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase02_avhrr_n16_n15', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n16', '2001-01-01', '2010-12-31', 'v01.2') w.add_secondary_sensor('avhrr-n15', '2001-01-01', '2010-12-31', 'v01.2') w.set_usecase_config('usecase-02.xml') w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow usecase = 'mms3' mmdtype = 'mmd3' w = Workflow(usecase) w.add_primary_sensor('atsr.1', '1991-08-01', '1996-09-01') w.add_primary_sensor('atsr.1', '1996-10-01', '1996-11-01') w.add_primary_sensor('atsr.1', '1996-12-30', '1997-02-01') w.add_primary_sensor('atsr.1', '1997-03-01', '1997-04-01') w.add_primary_sensor('atsr.1', '1997-05-01', '1997-06-01') w.add_primary_sensor('atsr.1', '1997-07-01', '1997-09-01') w.add_primary_sensor('atsr.1', '1997-10-01', '1997-11-01') w.add_primary_sensor('atsr.1', '1997-12-01', '1997-12-18') w.add_primary_sensor('atsr.2', '1995-06-01', '1996-01-01') w.add_primary_sensor('atsr.2', '1996-07-01', '2003-06-23') w.add_primary_sensor('atsr.3', '2002-05-20', '2012-04-09') w.set_samples_per_month(0) w.run(mmdtype, hosts=[('localhost', 24)], calls=[('sampling-start', 1)], with_history=True)
from workflow import Period from workflow import Workflow usecase = 'mms7' mmdtype = 'mmd7' w = Workflow(usecase, Period('2012-07-02', '2015-01-01')) w.add_primary_sensor('avhrr_f.m02', '2007-03-01', '2014-04-01') w.add_secondary_sensor('amsr2', '2012-07-02', '2015-04-01') w.set_samples_per_month(5000000) w.run(mmdtype, hosts=[('localhost', 12)], without_arc=True)
from workflow import Workflow w = Workflow('ingest_atsr_e1', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('atsr-e1', '1991-08-01', '1997-12-17', 'v3') w.run_ingestion(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase06c_sst_mbt_sst_amsre_aq', 7, '/group_workspaces/cems2/esacci_sst/mms_new/config') w.add_primary_sensor('mbt-sst', '2002-06-01', '2011-10-04', 'v03.3') w.add_secondary_sensor('amsre-aq', '2002-06-01', '2011-10-04', 'v12') w.set_usecase_config('usecase-06s-sst.xml') w.run_matchup(hosts=[('localhost', 24)])
from workflow import Workflow w = Workflow('usecase01_atsr_e2_avhrr_m02', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('atsr-e2', '2006-10-30', '2008-01-31', 'v3') w.add_secondary_sensor('avhrr-m02', '2006-10-30', '2008-01-31', 'v01.2') w.set_usecase_config('usecase-01.xml') w.run_matchup(hosts=[('localhost', 72)])
from workflow import Workflow w = Workflow('usecase02_avhrr_n19_iasi_mb', 7, '/group_workspaces/cems2/fiduceo/Software/mms/config') w.add_primary_sensor('avhrr-n19', '2013-02-20', '2015-12-31', 'v01.2') w.add_secondary_sensor('iasi-mb', '2013-02-20', '2015-12-31', 'latest') w.set_usecase_config('usecase-03.xml') w.run_matchup(hosts=[('localhost', 72)])