def test_simple_serial_vol_create(self):
        """Test that serially creating PVCs causes heketi to add volumes.
        """
        self.addCleanup(self.wait_to_settle)
        # TODO A nice thing to add to this test would be to also verify
        # the gluster volumes also exist.
        tname = make_unique_label(extract_method_name(self.id()))
        ocp_node = g.config['ocp_servers']['master'].keys()[0]
        # deploy a temporary storage class
        sc = build_storage_class(
            name=tname,
            resturl=self.heketi_server_url,
            restuser=self.heketi_cli_user,
            restuserkey=self.heketi_cli_key)
        with temp_config(ocp_node, sc) as tmpfn:
            oc_create(ocp_node, tmpfn)
        self.addCleanup(delete_storageclass, ocp_node, tname)
        orig_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))

        # deploy a persistent volume claim
        c1 = ClaimInfo(
            name='-'.join((tname, 'pvc1')),
            storageclass=tname,
            size=2)
        c1.create_pvc(ocp_node)
        self.addCleanup(c1.delete_pvc, ocp_node)
        c1.update_pvc_info(ocp_node)
        # verify volume exists
        self.assertTrue(c1.volumeName)
        c1.update_pv_info(ocp_node)
        self.assertTrue(c1.heketiVolumeName)

        # verify this is a new volume to heketi
        now_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))
        self.assertEqual(len(orig_vols) + 1, len(now_vols))
        self.assertIn(c1.heketiVolumeName, now_vols)
        self.assertNotIn(c1.heketiVolumeName, orig_vols)

        # deploy a 2nd pvc
        c2 = ClaimInfo(
            name='-'.join((tname, 'pvc2')),
            storageclass=tname,
            size=2)
        c2.create_pvc(ocp_node)
        self.addCleanup(c2.delete_pvc, ocp_node)
        c2.update_pvc_info(ocp_node)
        # verify volume exists
        self.assertTrue(c2.volumeName)
        c2.update_pv_info(ocp_node)
        self.assertTrue(c2.heketiVolumeName)

        # verify this is a new volume to heketi
        now_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))
        self.assertEqual(len(orig_vols) + 2, len(now_vols))
        self.assertIn(c2.heketiVolumeName, now_vols)
        self.assertNotIn(c2.heketiVolumeName, orig_vols)
    def test_multiple_vol_create(self):
        """Test creating two volumes via PVCs with no waiting between
        the PVC requests.

        We do wait after all the PVCs are submitted to get statuses.
        """
        self.addCleanup(self.wait_to_settle)
        tname = make_unique_label(extract_method_name(self.id()))
        ocp_node = g.config['ocp_servers']['master'].keys()[0]
        # deploy a temporary storage class
        sc = build_storage_class(
            name=tname,
            resturl=self.heketi_server_url,
            restuser=self.heketi_cli_user,
            restuserkey=self.heketi_cli_key)
        with temp_config(ocp_node, sc) as tmpfn:
            oc_create(ocp_node, tmpfn)
        self.addCleanup(delete_storageclass, ocp_node, tname)

        # deploy two persistent volume claims
        c1 = ClaimInfo(
            name='-'.join((tname, 'pvc1')),
            storageclass=tname,
            size=2)
        c1.create_pvc(ocp_node)
        self.addCleanup(c1.delete_pvc, ocp_node)
        c2 = ClaimInfo(
            name='-'.join((tname, 'pvc2')),
            storageclass=tname,
            size=2)
        c2.create_pvc(ocp_node)
        self.addCleanup(c2.delete_pvc, ocp_node)

        # wait for pvcs/volumes to complete
        c1.update_pvc_info(ocp_node)
        c2.update_pvc_info(ocp_node)
        now_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))

        # verify first volume exists
        self.assertTrue(c1.volumeName)
        c1.update_pv_info(ocp_node)
        self.assertTrue(c1.heketiVolumeName)
        # verify this volume in heketi
        self.assertIn(c1.heketiVolumeName, now_vols)

        # verify second volume exists
        self.assertTrue(c2.volumeName)
        c2.update_pv_info(ocp_node)
        self.assertTrue(c2.heketiVolumeName)
        # verify this volume in heketi
        self.assertIn(c2.heketiVolumeName, now_vols)
 def background_ops():
     subname = make_unique_label(short_tc_name)
     for i, w in enumerate(Waiter(60 * 60)):
         time.sleep(random.randint(1, 10) * 0.1)
         c = ClaimInfo(name='{}-{}'.format(subname, i),
                       storageclass=tname,
                       size=2)
         c.create_pvc(ocp_node)
         time.sleep(1)
         c.update_pvc_info(ocp_node, timeout=120)
         c.update_pv_info(ocp_node)
         time.sleep(random.randint(1, 10) * 0.1)
         c.delete_pvc(ocp_node)
         if done.is_set():
             break
    def test_threaded_multi_request(self, count):
        """Test creating volumes via PVCs where the pvc create
        commands are launched in parallell via threads.
        """
        self.addCleanup(self.wait_to_settle)
        tname = make_unique_label(extract_method_name(self.id()))
        ocp_node = g.config['ocp_servers']['master'].keys()[0]
        # deploy a temporary storage class
        sc = build_storage_class(
            name=tname,
            resturl=self.heketi_server_url,
            restuser=self.heketi_cli_user,
            restuserkey=self.heketi_cli_key)
        with temp_config(ocp_node, sc) as tmpfn:
            oc_create(ocp_node, tmpfn)
        self.addCleanup(delete_storageclass, ocp_node, tname)

        # prepare the persistent volume claims
        claims = [
            ClaimInfo(name='-'.join((tname, ('pvc%d' % n))),
                      storageclass=tname,
                      size=2)
            for n in range(count)]

        # create a "bunch" of pvc all at once
        def create(ci):
            ci.create_pvc(ocp_node)
            self.addCleanup(ci.delete_pvc, ocp_node)
        threads = [
            threading.Thread(target=create, args=[c])
            for c in claims]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

        for c in claims:
            c.update_pvc_info(ocp_node, timeout=120)
        now_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))
        for c in claims:
            c.update_pv_info(ocp_node)
            self.assertIn(c.heketiVolumeName, now_vols)
    def test_create_delete_volumes_concurrently(self):
        """Test creating volume when "other processes" are creating
        and deleting other volumes in the background.
        """
        self.addCleanup(self.wait_to_settle)
        tname = make_unique_label(extract_method_name(self.id()))
        ocp_node = g.config['ocp_servers']['master'].keys()[0]
        # deploy a temporary storage class
        sc = build_storage_class(name=tname,
                                 resturl=self.heketi_server_url,
                                 restuser=self.heketi_cli_user,
                                 restuserkey=self.heketi_cli_key)
        with temp_config(ocp_node, sc) as tmpfn:
            oc_create(ocp_node, tmpfn)
        self.addCleanup(delete_storageclass, ocp_node, tname)

        # make this a condition
        done = threading.Event()
        short_tc_name = "volumes-concurrently"

        def background_ops():
            subname = make_unique_label(short_tc_name)
            for i, w in enumerate(Waiter(60 * 60)):
                time.sleep(random.randint(1, 10) * 0.1)
                c = ClaimInfo(name='{}-{}'.format(subname, i),
                              storageclass=tname,
                              size=2)
                c.create_pvc(ocp_node)
                time.sleep(1)
                c.update_pvc_info(ocp_node, timeout=120)
                c.update_pv_info(ocp_node)
                time.sleep(random.randint(1, 10) * 0.1)
                c.delete_pvc(ocp_node)
                if done.is_set():
                    break

        failures = []

        def checked_background_ops():
            try:
                background_ops()
            except Exception as e:
                failures.append(e)

        count = 4
        threads = [
            threading.Thread(target=checked_background_ops)
            for _ in range(count)
        ]
        self.addCleanup(done.set)
        for t in threads:
            t.start()

        # let the threads start doing their own stuff
        time.sleep(10)

        # deploy two persistent volume claims
        c1 = ClaimInfo(name='-'.join((short_tc_name, 'pvc1')),
                       storageclass=tname,
                       size=2)
        c1.create_pvc(ocp_node)
        self.addCleanup(c1.delete_pvc, ocp_node)
        c2 = ClaimInfo(name='-'.join((short_tc_name, 'pvc2')),
                       storageclass=tname,
                       size=2)
        c2.create_pvc(ocp_node)
        self.addCleanup(c2.delete_pvc, ocp_node)

        # wait for pvcs/volumes to complete
        c1.update_pvc_info(ocp_node, timeout=120)
        c2.update_pvc_info(ocp_node, timeout=120)

        # verify first volume exists
        self.assertTrue(c1.volumeName)
        c1.update_pv_info(ocp_node)
        self.assertTrue(c1.heketiVolumeName)
        # verify this volume in heketi
        now_vols = _heketi_name_id_map(
            _heketi_vols(ocp_node, self.heketi_server_url))
        self.assertIn(c1.heketiVolumeName, now_vols)

        # verify second volume exists
        self.assertTrue(c2.volumeName)
        c2.update_pv_info(ocp_node)
        self.assertTrue(c2.heketiVolumeName)
        # verify this volume in heketi
        self.assertIn(c2.heketiVolumeName, now_vols)

        # clean up the background threads
        done.set()
        for t in threads:
            t.join()
        self.assertFalse(failures)