def test__constraints_for_new_request_no_new_data(self,
list_file_info_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
old_list = [
('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0,
119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0,
109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0,
113411),
]
config = {
'external_dataset_res': edres,
'new_data_check': old_list,
'ds_params': {
'base_url':
'test_data/ruv',
'list_pattern':
'RDLm_SEAB_*.ruv',
'date_pattern':
'%Y %m %d %H %M',
'date_extraction_pattern':
'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
list_file_info_mock.return_value = old_list
with self.assertRaises(NoNewDataWarning):
RuvDataHandler._constraints_for_new_request(config)
def test__constraints_for_new_request_no_new_data(self, list_file_info_mock):
edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
old_list = [('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0, 119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0, 109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0, 113411), ]
config = {
'external_dataset_res': edres,
'new_data_check': old_list,
'ds_params': {
'base_url': 'test_data/ruv',
'list_pattern': 'RDLm_SEAB_*.ruv',
'date_pattern': '%Y %m %d %H %M',
'date_extraction_pattern': 'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
list_file_info_mock.return_value = old_list
with self.assertRaises(NoNewDataWarning):
RuvDataHandler._constraints_for_new_request(config)
def test__constraints_for_new_request(self):
edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
# old_list = [
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0000.ruv',
# '04-Jun-2012 20:43',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0100.ruv',
# '04-Jun-2012 21:43',
# '135K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0200.ruv',
# '04-Jun-2012 22:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0300.ruv',
# '04-Jun-2012 23:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0400.ruv',
# '05-Jun-2012 00:41',
# '150K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0500.ruv',
# '05-Jun-2012 01:41',
# '142K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0600.ruv',
# '05-Jun-2012 02:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0700.ruv',
# '05-Jun-2012 03:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0800.ruv',
# '05-Jun-2012 04:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0900.ruv',
# '05-Jun-2012 05:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1000.ruv',
# '05-Jun-2012 06:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1100.ruv',
# '05-Jun-2012 07:40',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1200.ruv',
# '05-Jun-2012 08:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1300.ruv',
# '05-Jun-2012 09:39',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1400.ruv',
# '05-Jun-2012 10:38',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1500.ruv',
# '05-Jun-2012 11:43',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1600.ruv',
# '05-Jun-2012 12:43',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1700.ruv',
# '05-Jun-2012 13:42',
# '134K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1800.ruv',
# '05-Jun-2012 14:42',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1900.ruv',
# '05-Jun-2012 15:42',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2000.ruv',
# '05-Jun-2012 16:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2100.ruv',
# '05-Jun-2012 17:41',
# '160K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2200.ruv',
# '05-Jun-2012 18:41',
# '158K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2300.ruv',
# '05-Jun-2012 19:41',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0000.ruv',
# '05-Jun-2012 20:40',
# '140K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0100.ruv',
# '05-Jun-2012 21:40',
# '133K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0200.ruv',
# '05-Jun-2012 22:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0300.ruv',
# '05-Jun-2012 23:39',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0400.ruv',
# '06-Jun-2012 00:39',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0500.ruv',
# '06-Jun-2012 01:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0600.ruv',
# '06-Jun-2012 02:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0700.ruv',
# '06-Jun-2012 03:43',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0800.ruv',
# '06-Jun-2012 04:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0900.ruv',
# '06-Jun-2012 05:42',
# '130K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1000.ruv',
# '06-Jun-2012 06:42',
# '129K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1100.ruv',
# '06-Jun-2012 07:42',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1200.ruv',
# '06-Jun-2012 08:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1300.ruv',
# '06-Jun-2012 09:41',
# '151K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1400.ruv',
# '06-Jun-2012 10:41',
# '153K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1500.ruv',
# '06-Jun-2012 11:41',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1600.ruv',
# '06-Jun-2012 12:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1700.ruv',
# '06-Jun-2012 13:40',
# '161K'),]
old_list = [
('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0, 119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0, 109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0, 113411),
]
# old_list = None
config = {
'external_dataset_res':edres,
# 'new_data_check':None,
'new_data_check':old_list,
'ds_params':{
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
# 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/',
#CBM: What's the difference between RDLi and RDLm???
# 'pattern':'<a href="(RDLm.*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{1,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'base_url':'test_data/ruv',
'list_pattern':'RDLm_SEAB_*.ruv',
'date_pattern':'%Y %m %d %H %M',
'date_extraction_pattern': 'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
ret = RuvDataHandler._constraints_for_new_request(config)
log.debug('test__new_data_constraints: {0}'.format(config))
self.assertEqual(ret['new_files'], list_file_info(config['ds_params']['base_url'], config['ds_params']['list_pattern']))
def test__constraints_for_new_request(self):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# old_list = [
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0000.ruv',
# '04-Jun-2012 20:43',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0100.ruv',
# '04-Jun-2012 21:43',
# '135K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0200.ruv',
# '04-Jun-2012 22:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0300.ruv',
# '04-Jun-2012 23:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0400.ruv',
# '05-Jun-2012 00:41',
# '150K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0500.ruv',
# '05-Jun-2012 01:41',
# '142K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0600.ruv',
# '05-Jun-2012 02:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0700.ruv',
# '05-Jun-2012 03:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0800.ruv',
# '05-Jun-2012 04:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0900.ruv',
# '05-Jun-2012 05:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1000.ruv',
# '05-Jun-2012 06:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1100.ruv',
# '05-Jun-2012 07:40',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1200.ruv',
# '05-Jun-2012 08:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1300.ruv',
# '05-Jun-2012 09:39',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1400.ruv',
# '05-Jun-2012 10:38',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1500.ruv',
# '05-Jun-2012 11:43',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1600.ruv',
# '05-Jun-2012 12:43',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1700.ruv',
# '05-Jun-2012 13:42',
# '134K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1800.ruv',
# '05-Jun-2012 14:42',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1900.ruv',
# '05-Jun-2012 15:42',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2000.ruv',
# '05-Jun-2012 16:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2100.ruv',
# '05-Jun-2012 17:41',
# '160K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2200.ruv',
# '05-Jun-2012 18:41',
# '158K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2300.ruv',
# '05-Jun-2012 19:41',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0000.ruv',
# '05-Jun-2012 20:40',
# '140K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0100.ruv',
# '05-Jun-2012 21:40',
# '133K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0200.ruv',
# '05-Jun-2012 22:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0300.ruv',
# '05-Jun-2012 23:39',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0400.ruv',
# '06-Jun-2012 00:39',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0500.ruv',
# '06-Jun-2012 01:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0600.ruv',
# '06-Jun-2012 02:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0700.ruv',
# '06-Jun-2012 03:43',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0800.ruv',
# '06-Jun-2012 04:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0900.ruv',
# '06-Jun-2012 05:42',
# '130K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1000.ruv',
# '06-Jun-2012 06:42',
# '129K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1100.ruv',
# '06-Jun-2012 07:42',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1200.ruv',
# '06-Jun-2012 08:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1300.ruv',
# '06-Jun-2012 09:41',
# '151K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1400.ruv',
# '06-Jun-2012 10:41',
# '153K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1500.ruv',
# '06-Jun-2012 11:41',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1600.ruv',
# '06-Jun-2012 12:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1700.ruv',
# '06-Jun-2012 13:40',
# '161K'),]
old_list = [
('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0,
119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0,
109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0,
113411),
]
# old_list = None
config = {
'external_dataset_res': edres,
# 'new_data_check':None,
'new_data_check': old_list,
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
# 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/',
# CBM: What's the difference between RDLi and RDLm???
# 'pattern':'<a href="(RDLm.*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{1,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'base_url':
'test_data/ruv',
'list_pattern':
'RDLm_SEAB_*.ruv',
'date_pattern':
'%Y %m %d %H %M',
'date_extraction_pattern':
'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
ret = RuvDataHandler._constraints_for_new_request(config)
log.debug('test__new_data_constraints: {0}'.format(config))
self.assertEqual(
ret['new_files'],
list_file_info(config['ds_params']['base_url'],
config['ds_params']['list_pattern']))
