def setUp(self):
self.conf = get_config()['func_test']
if self.conf.get('auth_ssl', 'no') == 'yes':
auth_method = 'https://'
else:
auth_method = 'http://'
try:
self.conf = get_config()['func_test']
auth_host = self.conf.get('auth_host')
auth_port = self.conf.get('auth_port')
access_port = self.conf.get('access_port')
auth_url = self.conf.get('auth_prefix')
user_name = self.conf.get('username')
user_key = self.conf.get('password')
tenant_name = self.conf.get('account')
self.auth_token, self.account_id = get_auth(auth_host,
auth_port, auth_url, user_name, user_key, tenant_name)
self.os_access_root = '/v1/' + self.account_id
self.access_root = ('/' + self.conf.get('cdmi_root', 'cdmi') +
'/' + self.account_id)
self.cdmi_capability_root = ('/' +
self.conf.get('cdmi_root', 'cdmi') +
'/' +
self.account_id + '/' +
self.conf.get('cdmi_capability_id',
'cdmi_capabilities'))
#Setup two container names
suffix = format(time.time(), '.6f')
self.child_container = "cdmi_test_child_container_" + suffix
self.top_container = "cdmi_test_top_container_" + suffix
self.child_container_to_create = \
("cdmi_test_child_container_create_" + suffix)
self.top_container_to_create = ("cdmi_test_top_container_create_" +
suffix)
self.top_container_to_delete = ("cdmi_test_top_container_delete_" +
suffix)
#Create test containers
self.__create_test_container(self.top_container)
self.__create_test_container(self.top_container + '/' +
self.child_container)
self.__create_test_container(self.top_container_to_delete)
self.__create_test_container(self.top_container + "/a/b/c/d/e/f")
except Exception as login_error:
raise login_error
def setUp(self):
try:
config = get_config()
self.client = TestOpenStackClient(config.get('os_username'),
config.get('os_password'),
config.get('auth_url'))
self.client.tenant_name = config.get('os_tenant_name')
self._authenticate()
self.ns = 'http://schemas.dmtf.org/cimi/1'
self.nsmap = {'ns': self.ns}
self.host = config.get('api_url')
self.vol_host = config.get('vol_api_url')
self.baseURI = self.host + '/cimiv1'
self.image_id = self._prepare_id('images')
self.flavor_id = self._prepare_id('flavors')
self.volume_id = self._prepare_id('volumes')
if not self.volume_id:
self._create_volume()
self.server_id = self._prepare_id('servers')
if not self.server_id:
self._create_machine()
#self.server_id = self._prepare_id('servers')
except Exception as login_error:
raise login_error
def setUp(self):
try:
config = get_config()
self.client = TestOpenStackClient(config.get('os_username'),
config.get('os_password'),
config.get('auth_url'))
self.client.tenant_name = config.get('os_tenant_name')
self._authenticate()
self.ns = 'http://schemas.dmtf.org/cimi/1'
self.nsmap = {'ns':self.ns}
self.host = config.get('api_url')
self.vol_host = config.get('vol_api_url')
self.baseURI = self.host + '/cimiv1'
self.image_id = self._prepare_id('images')
self.flavor_id = self._prepare_id('flavors')
self.volume_id = self._prepare_id('volumes')
if not self.volume_id:
self._create_volume()
self.server_id = self._prepare_id('servers')
if not self.server_id:
self._create_machine()
#self.server_id = self._prepare_id('servers')
except Exception as login_error:
raise login_error
def setUp(self):
self.conf = get_config()['func_test']
if self.conf.get('auth_ssl', 'no') == 'yes':
auth_method = 'https://'
else:
auth_method = 'http://'
try:
self.conf = get_config()['func_test']
auth_host = self.conf.get('auth_host')
auth_port = self.conf.get('auth_port')
access_port = self.conf.get('access_port')
auth_url = self.conf.get('auth_prefix')
user_name = self.conf.get('username')
user_key = self.conf.get('password')
tenant_name = self.conf.get('account')
self.auth_token, self.account_id = get_auth(
auth_host, auth_port, auth_url, user_name, user_key,
tenant_name)
self.os_access_root = '/v1/' + self.account_id
self.access_root = ('/' + self.conf.get('cdmi_root', 'cdmi') +
'/' + self.account_id)
self.cdmi_capability_root = (
'/' + self.conf.get('cdmi_root', 'cdmi') + '/' +
self.account_id + '/' +
self.conf.get('cdmi_capability_id', 'cdmi_capabilities'))
#Setup two container names
suffix = format(time.time(), '.6f')
self.child_container = "cdmi_test_child_container_" + suffix
self.top_container = "cdmi_test_top_container_" + suffix
self.child_container_to_create = \
("cdmi_test_child_container_create_" + suffix)
self.top_container_to_create = ("cdmi_test_top_container_create_" +
suffix)
self.top_container_to_delete = ("cdmi_test_top_container_delete_" +
suffix)
#Create test containers
self.__create_test_container(self.top_container)
self.__create_test_container(self.top_container + '/' +
self.child_container)
self.__create_test_container(self.top_container_to_delete)
self.__create_test_container(self.top_container + "/a/b/c/d/e/f")
except Exception as login_error:
raise login_error
def test_sngp_script(self, dataset_name, classifier, representation_size,
correct_train_loss, correct_val_loss,
correct_fewshot_acc_sum, simulate_failure):
data_dir = self.data_dir
config = test_utils.get_config(dataset_name=dataset_name,
classifier=classifier,
representation_size=representation_size,
use_sngp=True,
use_gp_layer=True)
output_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
config.dataset_dir = data_dir
num_examples = config.batch_size * config.total_steps
if not simulate_failure:
# Check for any errors.
with tfds.testing.mock_data(num_examples=num_examples,
data_dir=data_dir):
train_loss, val_loss, fewshot_results = sngp.main(
config, output_dir)
else:
# Check for the ability to restart from a previous checkpoint (after
# failure, etc.).
output_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
# NOTE: Use this flag to simulate failing at a certain step.
config.testing_failure_step = config.total_steps - 1
config.checkpoint_steps = config.testing_failure_step
config.keep_checkpoint_steps = config.checkpoint_steps
with tfds.testing.mock_data(num_examples=num_examples,
data_dir=data_dir):
sngp.main(config, output_dir)
checkpoint_path = os.path.join(output_dir, 'checkpoint.npz')
self.assertTrue(os.path.exists(checkpoint_path))
checkpoint = checkpoint_utils.load_checkpoint(
None, checkpoint_path)
self.assertEqual(int(checkpoint['opt']['state']['step']),
config.testing_failure_step)
# This should resume from the failed step.
del config.testing_failure_step
with tfds.testing.mock_data(num_examples=num_examples,
data_dir=data_dir):
train_loss, val_loss, fewshot_results = sngp.main(
config, output_dir)
# Check for reproducibility.
fewshot_acc_sum = sum(jax.tree_util.tree_flatten(fewshot_results)[0])
logging.info('(train_loss, val_loss, fewshot_acc_sum) = %s, %s, %s',
train_loss, val_loss['val'], fewshot_acc_sum)
# TODO(dusenberrymw): Determine why the SNGP script is non-deterministic.
self.assertAllClose(train_loss,
correct_train_loss,
atol=0.025,
rtol=0.3)
self.assertAllClose(val_loss['val'],
correct_val_loss,
atol=0.02,
rtol=0.3)
def test_loading_pretrained_model(self, dataset_name, classifier,
representation_size, use_gp_layer,
correct_train_loss, correct_val_loss,
correct_fewshot_acc_sum,
finetune_dataset_name):
data_dir = self.data_dir
config = test_utils.get_config(dataset_name=dataset_name,
classifier=classifier,
representation_size=representation_size,
use_sngp=True,
use_gp_layer=use_gp_layer)
output_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
config.dataset_dir = data_dir
num_examples = config.batch_size * config.total_steps
# Run to save a checkpoint, then use that as a pretrained model.
output_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
with tfds.testing.mock_data(num_examples=num_examples,
data_dir=data_dir):
sngp.main(config, output_dir)
checkpoint_path = os.path.join(output_dir, 'checkpoint.npz')
self.assertTrue(os.path.exists(checkpoint_path))
output_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
config.model_init = checkpoint_path
config.model.representation_size = None
if finetune_dataset_name == 'cifar':
config.dataset = 'cifar10'
config.val_split = f'train[:{num_examples}]'
config.train_split = f'train[{num_examples}:{num_examples*2}]'
config.num_classes = 10
config.ood_datasets = ['cifar100']
config.ood_num_classes = [100]
config.ood_split = f'test[{num_examples*2}:{num_examples*3}]'
config.ood_methods = ['maha', 'entropy', 'rmaha', 'msp']
config.eval_on_cifar_10h = True
config.cifar_10h_split = f'test[:{num_examples}]'
config.pp_eval_cifar_10h = (
'decode|resize(384)|value_range(-1, 1)|keep(["image", "labels"])'
)
elif finetune_dataset_name == 'imagenet':
config.dataset = 'imagenet2012'
config.val_split = f'train[:{num_examples}]'
config.train_split = f'train[{num_examples}:{num_examples*2}]'
config.num_classes = 1000
pp_common = '|value_range(-1, 1)'
pp_common += f'|onehot({config.num_classes}, key="label", key_result="labels")' # pylint: disable=line-too-long
pp_common += '|keep(["image", "labels"])'
config.pp_train = 'decode|resize_small(512)|random_crop(384)' + pp_common
config.pp_eval = 'decode|resize(384)' + pp_common
config.fewshot.pp_train = 'decode|resize_small(512)|central_crop(384)|value_range(-1,1)|drop("segmentation_mask")'
config.fewshot.pp_eval = 'decode|resize(384)|value_range(-1,1)|drop("segmentation_mask")'
if config.get('ood_num_classes'):
pp_eval_ood = []
for num_classes in config.ood_num_classes:
pp_eval_ood.append(
config.pp_eval.replace(f'onehot({config.num_classes}',
f'onehot({num_classes}'))
config.pp_eval_ood = pp_eval_ood
with tfds.testing.mock_data(num_examples=num_examples,
data_dir=data_dir):
logging.info('!!!config %s', config)
train_loss, val_loss, fewshot_results = sngp.main(
config, output_dir)
fewshot_acc_sum = sum(jax.tree_util.tree_flatten(fewshot_results)[0])
logging.info('(train_loss, val_loss, fewshot_acc_sum) = %s, %s, %s',
train_loss, val_loss['val'], fewshot_acc_sum)
# TODO(dusenberrymw,jjren): Add a reproducibility test for OOD eval.
# TODO(dusenberrymw): Determine why the SNGP script is non-deterministic.
self.assertAllClose(train_loss,
correct_train_loss,
atol=1e-3,
rtol=1e-3)
self.assertAllClose(val_loss['val'],
correct_val_loss,
atol=2e-3,
rtol=2e-3)
#TODO: test custom and canned acls that are both valid an invalid
def test_object_torrent(self):
"""Tests object torrents"""
self.fail("Feature not implemented yet")
def tearDown(self):
"""Tearing down the env after a test"""
print_header("Cleaning up the test bucket: " + self.test_bucket_name)
clean_bucket(self.test_bucket)
self.test_bucket = None
def testName(self):
pass
def suite():
#tests = ['test_object_basic_ops','test_object_versioning_enabled', 'test_object_versioning_suspended', 'test_object_acl', 'test_object_torrent']
tests = [
'test_object_basic_ops', 'test_object_versioning_enabled',
'test_object_versionlisting'
]
return unittest.TestSuite(map(ObjectTest, tests))
if __name__ == "__main__":
config = get_config()
suite = suite()
unittest.TextTestRunner(verbosity=2).run(suite)
#unittest.main()
def test_object_acl(self):
"""Tests object acl get/set and manipulation"""
self.fail("Test not implemented")
#TODO: test custom and canned acls that are both valid an invalid
def test_object_torrent(self):
"""Tests object torrents"""
self.fail("Feature not implemented yet")
def tearDown(self):
"""Tearing down the env after a test"""
print_header("Cleaning up the test bucket: " + self.test_bucket_name)
clean_bucket(self.test_bucket)
self.test_bucket = None
def testName(self):
pass
def suite():
#tests = ['test_object_basic_ops','test_object_versioning_enabled', 'test_object_versioning_suspended', 'test_object_acl', 'test_object_torrent']
tests = ['test_object_basic_ops', 'test_object_versioning_enabled', 'test_object_versionlisting']
return unittest.TestSuite(map(ObjectTest, tests))
if __name__ == "__main__":
config = get_config()
suite = suite()
unittest.TextTestRunner(verbosity=2).run(suite)
#unittest.main()
print "Deleting the bucket"
self.walrus.delete_bucket(testbucket)
def test_list_multipart_uploads(self):
self.fail("Feature Not implemented")
def test_bucket_lifecycle(self):
self.fail("Feature Not implemented")
def test_bucket_policy(self):
self.fail("Feature Not implemented")
def test_bucket_website(self):
self.fail("FeatureNot implemented")
def testName(self):
print "BucketTest"
return "BucketTest"
def suite():
#tests = ['test_bucket_get_put_delete', 'test_bucket_acl', 'test_bucket_versioning', 'test_bucket_logging', 'test_bucket_location', 'test_bucket_key_listing_paging', 'test_bucket_key_list_delim_prefix' ]
tests = ['test_bucket_get_put_delete', 'test_bucket_acl', 'test_bucket_versioning', 'test_bucket_location']
return unittest.TestSuite(map(BucketTest, tests))
if __name__ == "__main__":
config = test_utils.get_config()
suite = suite()
unittest.TextTestRunner(verbosity=2).run(suite)
#unittest.main()
