def setUp(self): self.conf = get_config()['func_test'] if self.conf.get('auth_ssl', 'no') == 'yes': auth_method = 'https://' else: auth_method = 'http://' try: self.conf = get_config()['func_test'] auth_host = self.conf.get('auth_host') auth_port = self.conf.get('auth_port') access_port = self.conf.get('access_port') auth_url = self.conf.get('auth_prefix') user_name = self.conf.get('username') user_key = self.conf.get('password') tenant_name = self.conf.get('account') self.auth_token, self.account_id = get_auth(auth_host, auth_port, auth_url, user_name, user_key, tenant_name) self.os_access_root = '/v1/' + self.account_id self.access_root = ('/' + self.conf.get('cdmi_root', 'cdmi') + '/' + self.account_id) self.cdmi_capability_root = ('/' + self.conf.get('cdmi_root', 'cdmi') + '/' + self.account_id + '/' + self.conf.get('cdmi_capability_id', 'cdmi_capabilities')) #Setup two container names suffix = format(time.time(), '.6f') self.child_container = "cdmi_test_child_container_" + suffix self.top_container = "cdmi_test_top_container_" + suffix self.child_container_to_create = \ ("cdmi_test_child_container_create_" + suffix) self.top_container_to_create = ("cdmi_test_top_container_create_" + suffix) self.top_container_to_delete = ("cdmi_test_top_container_delete_" + suffix) #Create test containers self.__create_test_container(self.top_container) self.__create_test_container(self.top_container + '/' + self.child_container) self.__create_test_container(self.top_container_to_delete) self.__create_test_container(self.top_container + "/a/b/c/d/e/f") except Exception as login_error: raise login_error
def setUp(self): try: config = get_config() self.client = TestOpenStackClient(config.get('os_username'), config.get('os_password'), config.get('auth_url')) self.client.tenant_name = config.get('os_tenant_name') self._authenticate() self.ns = 'http://schemas.dmtf.org/cimi/1' self.nsmap = {'ns': self.ns} self.host = config.get('api_url') self.vol_host = config.get('vol_api_url') self.baseURI = self.host + '/cimiv1' self.image_id = self._prepare_id('images') self.flavor_id = self._prepare_id('flavors') self.volume_id = self._prepare_id('volumes') if not self.volume_id: self._create_volume() self.server_id = self._prepare_id('servers') if not self.server_id: self._create_machine() #self.server_id = self._prepare_id('servers') except Exception as login_error: raise login_error
def setUp(self): try: config = get_config() self.client = TestOpenStackClient(config.get('os_username'), config.get('os_password'), config.get('auth_url')) self.client.tenant_name = config.get('os_tenant_name') self._authenticate() self.ns = 'http://schemas.dmtf.org/cimi/1' self.nsmap = {'ns':self.ns} self.host = config.get('api_url') self.vol_host = config.get('vol_api_url') self.baseURI = self.host + '/cimiv1' self.image_id = self._prepare_id('images') self.flavor_id = self._prepare_id('flavors') self.volume_id = self._prepare_id('volumes') if not self.volume_id: self._create_volume() self.server_id = self._prepare_id('servers') if not self.server_id: self._create_machine() #self.server_id = self._prepare_id('servers') except Exception as login_error: raise login_error
def setUp(self): self.conf = get_config()['func_test'] if self.conf.get('auth_ssl', 'no') == 'yes': auth_method = 'https://' else: auth_method = 'http://' try: self.conf = get_config()['func_test'] auth_host = self.conf.get('auth_host') auth_port = self.conf.get('auth_port') access_port = self.conf.get('access_port') auth_url = self.conf.get('auth_prefix') user_name = self.conf.get('username') user_key = self.conf.get('password') tenant_name = self.conf.get('account') self.auth_token, self.account_id = get_auth( auth_host, auth_port, auth_url, user_name, user_key, tenant_name) self.os_access_root = '/v1/' + self.account_id self.access_root = ('/' + self.conf.get('cdmi_root', 'cdmi') + '/' + self.account_id) self.cdmi_capability_root = ( '/' + self.conf.get('cdmi_root', 'cdmi') + '/' + self.account_id + '/' + self.conf.get('cdmi_capability_id', 'cdmi_capabilities')) #Setup two container names suffix = format(time.time(), '.6f') self.child_container = "cdmi_test_child_container_" + suffix self.top_container = "cdmi_test_top_container_" + suffix self.child_container_to_create = \ ("cdmi_test_child_container_create_" + suffix) self.top_container_to_create = ("cdmi_test_top_container_create_" + suffix) self.top_container_to_delete = ("cdmi_test_top_container_delete_" + suffix) #Create test containers self.__create_test_container(self.top_container) self.__create_test_container(self.top_container + '/' + self.child_container) self.__create_test_container(self.top_container_to_delete) self.__create_test_container(self.top_container + "/a/b/c/d/e/f") except Exception as login_error: raise login_error
def test_sngp_script(self, dataset_name, classifier, representation_size, correct_train_loss, correct_val_loss, correct_fewshot_acc_sum, simulate_failure): data_dir = self.data_dir config = test_utils.get_config(dataset_name=dataset_name, classifier=classifier, representation_size=representation_size, use_sngp=True, use_gp_layer=True) output_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) config.dataset_dir = data_dir num_examples = config.batch_size * config.total_steps if not simulate_failure: # Check for any errors. with tfds.testing.mock_data(num_examples=num_examples, data_dir=data_dir): train_loss, val_loss, fewshot_results = sngp.main( config, output_dir) else: # Check for the ability to restart from a previous checkpoint (after # failure, etc.). output_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) # NOTE: Use this flag to simulate failing at a certain step. config.testing_failure_step = config.total_steps - 1 config.checkpoint_steps = config.testing_failure_step config.keep_checkpoint_steps = config.checkpoint_steps with tfds.testing.mock_data(num_examples=num_examples, data_dir=data_dir): sngp.main(config, output_dir) checkpoint_path = os.path.join(output_dir, 'checkpoint.npz') self.assertTrue(os.path.exists(checkpoint_path)) checkpoint = checkpoint_utils.load_checkpoint( None, checkpoint_path) self.assertEqual(int(checkpoint['opt']['state']['step']), config.testing_failure_step) # This should resume from the failed step. del config.testing_failure_step with tfds.testing.mock_data(num_examples=num_examples, data_dir=data_dir): train_loss, val_loss, fewshot_results = sngp.main( config, output_dir) # Check for reproducibility. fewshot_acc_sum = sum(jax.tree_util.tree_flatten(fewshot_results)[0]) logging.info('(train_loss, val_loss, fewshot_acc_sum) = %s, %s, %s', train_loss, val_loss['val'], fewshot_acc_sum) # TODO(dusenberrymw): Determine why the SNGP script is non-deterministic. self.assertAllClose(train_loss, correct_train_loss, atol=0.025, rtol=0.3) self.assertAllClose(val_loss['val'], correct_val_loss, atol=0.02, rtol=0.3)
def test_loading_pretrained_model(self, dataset_name, classifier, representation_size, use_gp_layer, correct_train_loss, correct_val_loss, correct_fewshot_acc_sum, finetune_dataset_name): data_dir = self.data_dir config = test_utils.get_config(dataset_name=dataset_name, classifier=classifier, representation_size=representation_size, use_sngp=True, use_gp_layer=use_gp_layer) output_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) config.dataset_dir = data_dir num_examples = config.batch_size * config.total_steps # Run to save a checkpoint, then use that as a pretrained model. output_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) with tfds.testing.mock_data(num_examples=num_examples, data_dir=data_dir): sngp.main(config, output_dir) checkpoint_path = os.path.join(output_dir, 'checkpoint.npz') self.assertTrue(os.path.exists(checkpoint_path)) output_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) config.model_init = checkpoint_path config.model.representation_size = None if finetune_dataset_name == 'cifar': config.dataset = 'cifar10' config.val_split = f'train[:{num_examples}]' config.train_split = f'train[{num_examples}:{num_examples*2}]' config.num_classes = 10 config.ood_datasets = ['cifar100'] config.ood_num_classes = [100] config.ood_split = f'test[{num_examples*2}:{num_examples*3}]' config.ood_methods = ['maha', 'entropy', 'rmaha', 'msp'] config.eval_on_cifar_10h = True config.cifar_10h_split = f'test[:{num_examples}]' config.pp_eval_cifar_10h = ( 'decode|resize(384)|value_range(-1, 1)|keep(["image", "labels"])' ) elif finetune_dataset_name == 'imagenet': config.dataset = 'imagenet2012' config.val_split = f'train[:{num_examples}]' config.train_split = f'train[{num_examples}:{num_examples*2}]' config.num_classes = 1000 pp_common = '|value_range(-1, 1)' pp_common += f'|onehot({config.num_classes}, key="label", key_result="labels")' # pylint: disable=line-too-long pp_common += '|keep(["image", "labels"])' config.pp_train = 'decode|resize_small(512)|random_crop(384)' + pp_common config.pp_eval = 'decode|resize(384)' + pp_common config.fewshot.pp_train = 'decode|resize_small(512)|central_crop(384)|value_range(-1,1)|drop("segmentation_mask")' config.fewshot.pp_eval = 'decode|resize(384)|value_range(-1,1)|drop("segmentation_mask")' if config.get('ood_num_classes'): pp_eval_ood = [] for num_classes in config.ood_num_classes: pp_eval_ood.append( config.pp_eval.replace(f'onehot({config.num_classes}', f'onehot({num_classes}')) config.pp_eval_ood = pp_eval_ood with tfds.testing.mock_data(num_examples=num_examples, data_dir=data_dir): logging.info('!!!config %s', config) train_loss, val_loss, fewshot_results = sngp.main( config, output_dir) fewshot_acc_sum = sum(jax.tree_util.tree_flatten(fewshot_results)[0]) logging.info('(train_loss, val_loss, fewshot_acc_sum) = %s, %s, %s', train_loss, val_loss['val'], fewshot_acc_sum) # TODO(dusenberrymw,jjren): Add a reproducibility test for OOD eval. # TODO(dusenberrymw): Determine why the SNGP script is non-deterministic. self.assertAllClose(train_loss, correct_train_loss, atol=1e-3, rtol=1e-3) self.assertAllClose(val_loss['val'], correct_val_loss, atol=2e-3, rtol=2e-3)
#TODO: test custom and canned acls that are both valid an invalid def test_object_torrent(self): """Tests object torrents""" self.fail("Feature not implemented yet") def tearDown(self): """Tearing down the env after a test""" print_header("Cleaning up the test bucket: " + self.test_bucket_name) clean_bucket(self.test_bucket) self.test_bucket = None def testName(self): pass def suite(): #tests = ['test_object_basic_ops','test_object_versioning_enabled', 'test_object_versioning_suspended', 'test_object_acl', 'test_object_torrent'] tests = [ 'test_object_basic_ops', 'test_object_versioning_enabled', 'test_object_versionlisting' ] return unittest.TestSuite(map(ObjectTest, tests)) if __name__ == "__main__": config = get_config() suite = suite() unittest.TextTestRunner(verbosity=2).run(suite) #unittest.main()
def test_object_acl(self): """Tests object acl get/set and manipulation""" self.fail("Test not implemented") #TODO: test custom and canned acls that are both valid an invalid def test_object_torrent(self): """Tests object torrents""" self.fail("Feature not implemented yet") def tearDown(self): """Tearing down the env after a test""" print_header("Cleaning up the test bucket: " + self.test_bucket_name) clean_bucket(self.test_bucket) self.test_bucket = None def testName(self): pass def suite(): #tests = ['test_object_basic_ops','test_object_versioning_enabled', 'test_object_versioning_suspended', 'test_object_acl', 'test_object_torrent'] tests = ['test_object_basic_ops', 'test_object_versioning_enabled', 'test_object_versionlisting'] return unittest.TestSuite(map(ObjectTest, tests)) if __name__ == "__main__": config = get_config() suite = suite() unittest.TextTestRunner(verbosity=2).run(suite) #unittest.main()
print "Deleting the bucket" self.walrus.delete_bucket(testbucket) def test_list_multipart_uploads(self): self.fail("Feature Not implemented") def test_bucket_lifecycle(self): self.fail("Feature Not implemented") def test_bucket_policy(self): self.fail("Feature Not implemented") def test_bucket_website(self): self.fail("FeatureNot implemented") def testName(self): print "BucketTest" return "BucketTest" def suite(): #tests = ['test_bucket_get_put_delete', 'test_bucket_acl', 'test_bucket_versioning', 'test_bucket_logging', 'test_bucket_location', 'test_bucket_key_listing_paging', 'test_bucket_key_list_delim_prefix' ] tests = ['test_bucket_get_put_delete', 'test_bucket_acl', 'test_bucket_versioning', 'test_bucket_location'] return unittest.TestSuite(map(BucketTest, tests)) if __name__ == "__main__": config = test_utils.get_config() suite = suite() unittest.TextTestRunner(verbosity=2).run(suite) #unittest.main()