def test__init_acquisition_cycle_ext_ds_res(self, SlocumParser_mock):
edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
edres.dataset_description.parameters['dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat'
config = {'external_dataset_res':edres}
SlocumDataHandler._init_acquisition_cycle(config)
SlocumParser_mock.assert_called_once_with(edres.dataset_description.parameters['dataset_path'])
self.assertIn('parser', config)
self.assertEquals(config['parser'],SlocumParser_mock())
def test__init_acquisition_cycle_ext_ds_res(self, SlocumParser_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat'
config = {'external_dataset_res': edres}
SlocumDataHandler._init_acquisition_cycle(config)
SlocumParser_mock.assert_called_once_with(
edres.dataset_description.parameters['dataset_path'])
self.assertIn('parser', config)
self.assertEquals(config['parser'], SlocumParser_mock())
def test__constraints_for_new_request(self):
# ret = SlocumDataHandler._constraints_for_new_request({})
# self.assertIsInstance(ret, dict)
old_list = [
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0, 521081),
('test_data/slocum/ru05-2012-022-0-0-sbd.dat', 1337261358.0, 521081),
]
# old_list = None
edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
config = {
'external_dataset_res':edres,
'new_data_check':old_list,
'ds_params':{
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
'base_url':'test_data/slocum/',
'list_pattern':'ru05-*-sbd.dat',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'date_pattern':'%Y %j',
'date_extraction_pattern':'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
}
}
ret = SlocumDataHandler._constraints_for_new_request(config)
log.debug('test__constraints_for_new_request: {0}'.format(ret['new_files']))
self.assertEqual(ret['new_files'], list_file_info(config['ds_params']['base_url'], config['ds_params']['list_pattern']))
def test__get_data(self, RecordDictionaryTool_mock):
config = {
'constraints': {
'new_files': [
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0, 521081), ]
},
'param_dictionary': sentinel.pdict,
'data_producer_id': sentinel.dprod_id,
'stream_def': sentinel.stream_def_id
}
my_dict = {}
def setitem(name, val):
my_dict[name] = val
retval = MagicMock(spec=RecordDictionaryTool)
retval.__setitem__ = Mock(side_effect=setitem)
retval.to_granule.return_value = MagicMock(spec=Granule)
RecordDictionaryTool_mock.return_value = retval
for x in SlocumDataHandler._get_data(config):
self.assertTrue(isinstance(x, Granule))
retval.to_granule.assert_any_call()
log.debug(x)
def test__get_data(self, RecordDictionaryTool_mock):
config = {
'constraints': {
'new_files': [
('test_data/slocum/ru05-2012-021-0-0-sbd.dat',
1337261358.0, 521081),
]
},
'param_dictionary': sentinel.pdict,
'data_producer_id': sentinel.dprod_id,
'stream_def': sentinel.stream_def_id
}
my_dict = {}
def setitem(name, val):
my_dict[name] = val
retval = MagicMock(spec=RecordDictionaryTool)
retval.__setitem__ = Mock(side_effect=setitem)
retval.to_granule.return_value = MagicMock(spec=Granule)
RecordDictionaryTool_mock.return_value = retval
for x in SlocumDataHandler._get_data(config):
self.assertTrue(isinstance(x, Granule))
retval.to_granule.assert_any_call()
log.debug(x)
def test__constraints_for_historical_request(self):
config = {
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
# 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/',
# 'pattern':'<a href="([^"]*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{3,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'base_url': 'test_data/slocum',
'list_pattern':
'ru05-*-sbd.dat', # Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'date_pattern': '%Y %j',
'date_extraction_pattern':
'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
},
'constraints': {
'start_time': 1327122000,
'end_time': 1327294800
}
}
ret = SlocumDataHandler._constraints_for_historical_request(config)
log.debug(
'test_constraints_for_historical_request: {0}'.format(config))
self.assertEqual(
ret['new_files'],
list_file_info(config['ds_params']['base_url'],
config['ds_params']['list_pattern']))
def test__init_acquisition_cycle_ext_ds_res(self):
edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
edres.dataset_description.parameters['base_url'] = 'test_data/dir'
edres.dataset_description.parameters['header_count'] = 12
edres.dataset_description.parameters['pattern'] = 'test_filter'
config = {'external_dataset_res':edres}
SlocumDataHandler._init_acquisition_cycle(config)
self.assertIn('ds_params', config)
ds_params = config['ds_params']
self.assertIn('header_count', ds_params)
self.assertEquals(ds_params['header_count'],12)
self.assertIn('base_url',ds_params)
self.assertEquals(ds_params['base_url'],'test_data/dir')
self.assertIn('pattern',ds_params)
self.assertEquals(ds_params['pattern'], 'test_filter')
def test__get_data(self):
config = {
'constraints':{
'new_files':[
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0, 521081),
]
}
}
for x in SlocumDataHandler._get_data(config):
log.debug(x)
def test__init_acquisition_cycle_ext_ds_res(self):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters['base_url'] = 'test_data/dir'
edres.dataset_description.parameters['header_count'] = 12
edres.dataset_description.parameters['pattern'] = 'test_filter'
config = {'external_dataset_res': edres}
SlocumDataHandler._init_acquisition_cycle(config)
self.assertIn('ds_params', config)
ds_params = config['ds_params']
self.assertIn('header_count', ds_params)
self.assertEquals(ds_params['header_count'], 12)
self.assertIn('base_url', ds_params)
self.assertEquals(ds_params['base_url'], 'test_data/dir')
self.assertIn('pattern', ds_params)
self.assertEquals(ds_params['pattern'], 'test_filter')
def test__constraints_for_historical_request(self):
config = {
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
# 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/',
# 'pattern':'<a href="([^"]*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{3,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'base_url': 'test_data/slocum',
'list_pattern': 'ru05-*-sbd.dat', # Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'date_pattern': '%Y %j',
'date_extraction_pattern': 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'},
'constraints': {
'start_time': 1327122000,
'end_time': 1327294800
}
}
ret = SlocumDataHandler._constraints_for_historical_request(config)
log.debug('test_constraints_for_historical_request: {0}'.format(config))
self.assertEqual(ret['new_files'], list_file_info(config['ds_params']['base_url'], config['ds_params']['list_pattern']))
def test__get_data(self, build_granule_mock, ParamDict_mock, RecordDictionaryTool_mock):
config = {
'constraints':{
'new_files':[
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0, 521081),
]
},
'param_dictionary':sentinel.pdict,
'data_producer_id':sentinel.dprod_id
}
ParamDict_mock.load.return_value = sentinel.pdict_return_value
RecordDictionaryTool_mock.side_effect = lambda **kwargs: {}
for x in SlocumDataHandler._get_data(config):
RecordDictionaryTool_mock.assert_called_with(param_dictionary=sentinel.pdict_return_value)
build_granule_mock.assert_called_with(data_producer_id=sentinel.dprod_id, record_dictionary=ANY, param_dictionary=sentinel.pdict_return_value)
cargs=build_granule_mock.call_args
self.assertIn('c_wpt_y_lmc', cargs[1]['record_dictionary'])
self.assertIsInstance(cargs[1]['record_dictionary']['c_wpt_y_lmc'], numpy.ndarray)
log.debug(x)
def test__constraints_for_new_request(self):
# ret = SlocumDataHandler._constraints_for_new_request({})
# self.assertIsInstance(ret, dict)
old_list = [
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0,
521081),
('test_data/slocum/ru05-2012-022-0-0-sbd.dat', 1337261358.0,
521081),
]
# old_list = None
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
config = {
'external_dataset_res': edres,
'new_data_check': old_list,
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
'base_url': 'test_data/slocum/',
'list_pattern':
'ru05-*-sbd.dat', # Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'date_pattern': '%Y %j',
'date_extraction_pattern':
'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
}
}
ret = SlocumDataHandler._constraints_for_new_request(config)
log.debug('test__constraints_for_new_request: {0}'.format(
ret['new_files']))
self.assertEqual(
ret['new_files'],
list_file_info(config['ds_params']['base_url'],
config['ds_params']['list_pattern']))
def test__new_data_constraints(self):
ret = SlocumDataHandler._new_data_constraints({})
self.assertIsInstance(ret, dict)
def test__init_acquisition_cycle_no_ext_ds_res(self):
config = {}
SlocumDataHandler._init_acquisition_cycle(config)
self.assertEqual(config,config)
def test__new_data_constraints(self):
ret = SlocumDataHandler._new_data_constraints({})
self.assertIsInstance(ret, dict)
def test__init_acquisition_cycle_no_ext_ds_res(self):
config = {}
SlocumDataHandler._init_acquisition_cycle(config)
self.assertEqual(config, config)
