def load_datasets_from_config(extra_opts, *loader_opts):
'''
Generic dataset loading function.
'''
for opt in loader_opts:
loader_name = opt['loader_name']
if loader_name == 'esgf':
if extra_opts['password'] is None:
extra_opts['username'] = raw_input('Enter your ESGF OpenID:\n')
extra_opts['password'] = getpass(
prompt='Enter your ESGF password:\n')
opt['esgf_username'] = extra_opts['username']
opt['esgf_password'] = extra_opts['password']
elif loader_name == 'rcmed':
opt['min_lat'] = extra_opts['min_lat']
opt['max_lat'] = extra_opts['max_lat']
opt['min_lon'] = extra_opts['min_lon']
opt['max_lon'] = extra_opts['max_lon']
opt['start_time'] = extra_opts['start_time']
opt['end_time'] = extra_opts['end_time']
loader = DatasetLoader(*loader_opts)
loader.load_datasets()
return loader.datasets
def testExistingDataSource(self):
'''
Ensures that existing data source loaders can be added
'''
self.loader = DatasetLoader(self.config)
self.loader.load_datasets()
self.assertEqual(self.loader.datasets[0].origin['source'], 'local')
np.testing.assert_array_equal(self.loader.datasets[0].values,
self.values)
def testNewDataSource(self):
'''
Ensures that custom data source loaders can be added
'''
self.loader = DatasetLoader(self.new_data_source_config)
# Here the data_source "foo" represents the Dataset constructor
self.loader.add_source_loader('foo', build_dataset)
self.loader.load_datasets()
self.assertEqual(self.loader.datasets[0].origin['source'], 'foo')
np.testing.assert_array_equal(self.loader.datasets[0].values,
self.values2)
def testMultipleDataSources(self):
'''
Test for when multiple dataset configs are specified
'''
self.loader = DatasetLoader(self.config, self.new_data_source_config)
# Here the data_source "foo" represents the Dataset constructor
self.loader.add_source_loader('foo', build_dataset)
self.loader.load_datasets()
self.assertEqual(self.loader.datasets[0].origin['source'], 'local')
self.assertEqual(self.loader.datasets[1].origin['source'], 'foo')
np.testing.assert_array_equal(self.loader.datasets[0].values,
self.values)
np.testing.assert_array_equal(self.loader.datasets[1].values,
self.values2)
