def test_filecoll_extend(self, mock_api):
        """Test extend"""

        col = FileCollection(mock_api)
        col2 = FileCollection(mock_api)

        test_file = mock.create_autospec(UserFile)
        test_file2 = mock.create_autospec(UserFile)

        col._collection = [test_file]
        col2._collection = [test_file2, test_file]

        with self.assertRaises(AttributeError):
            col.extend(None)
        with self.assertRaises(AttributeError):
            col.extend("test")
        with self.assertRaises(AttributeError):
            col.extend([])

        col.extend(col2)
        self.assertEqual(len(col._collection), 2)
        self.assertTrue(all(i in [test_file, test_file2]
                            for i in col._collection))

        col2.extend(col)
        self.assertEqual(len(col._collection), 2)
        self.assertTrue(all(i in [test_file, test_file2]
                            for i in col._collection))
    def test_filecoll_remove(self, mock_api):
        """Test remove"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"
        col._collection = [test_file, 1, "2", None, []]

        with self.assertRaises(TypeError):
            col.remove(None)
        with self.assertRaises(TypeError):
            col.remove(10)

        col.remove(1)
        col.remove(-1)
        col.remove(slice(1))
        self.assertEqual(col._collection, ["2", None])

        test_file2 = mock.create_autospec(UserFile)
        test_file2.name = "test2"
        test_file3 = mock.create_autospec(UserFile)
        test_file3.name = "test3"
        col._collection = [test_file, test_file2, test_file3]
        col.remove("test")
        self.assertEqual(col._collection, [test_file2, test_file3])
        col.remove(["test2", "test3"])
        self.assertEqual(col._collection, [])
Exemple #3
0
    def add_file_collection(self, file_collection=None):
        """
        Assign a file collection to the job.
        The userfiles assigned to a job submission will be loaded onto each
        node prior to the job being executed.

        :Kwargs:
            - file_collection (:py:class:`.FileCollection`): If set, this will
              be assigned as the :py:class:`.FileCollection` of the job. If
              not set, a new :py:class:`.FileCollection` will be created.

        :Raises:
            - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
              passed in.
        """
        if file_collection is None:
            self._log.info("Assigning empty FileCollection to job")
            self.required_files = FileCollection(self._api)

        elif hasattr(file_collection, 'add'):
            self._log.debug("Assigning FileCollection with {0} "
                            "userfiles to job".format(len(file_collection)))

            self.required_files = file_collection

        else:
            raise TypeError("Can only assign an object of type FileCollection"
                            ", not {type}".format(type=type(file_collection)))
    def test_filecoll_add(self, mock_api):
        """Test add"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)

        with self.assertRaises(FileInvalidException):
            col.add("test")
        with self.assertRaises(FileInvalidException):
            col.add(1)
        with self.assertRaises(FileInvalidException):
            col.add(None)

        col.add(test_file)
        self.assertEqual(col._collection, [test_file])
        with self.assertRaises(FileInvalidException):
            col.add(test_file)

        col._collection = []
        col.add([test_file])
        self.assertEqual(col._collection, [test_file])
        col.add([test_file])
        self.assertEqual(col._collection, [test_file])

        col._collection = []
        col.add([test_file, test_file])

        col._collection = []
        col.add([1, "2", None])
        self.assertEqual(col._collection, [])
    def test_filecoll_str(self, mock_api, mock_add):
        """Test __str__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        colstr = str(col)
        self.assertEqual(colstr, "['1', 'None', 'test', '[]', '{}']")
    def test_filecoll_len(self, mock_api, mock_add):
        """Test __len__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        self.assertEqual(len(col), len(col._collection))

        col._collection.append("more")
        self.assertEqual(len(col), len(col._collection))
    def test_filecoll_is_uploaded(self,
                                  mock_rem,
                                  mock_mess,
                                  mock_ufile,
                                  mock_api):
        """Test is_uploaded"""

        def user_file_gen(u_name):
            """Mock UserFile generator"""
            ugen = mock.create_autospec(UserFile)
            ugen.name = str(u_name)
            ugen.compare_lastmodified.return_value = True
            return ugen

        def add(col, itm):
            """Mock add UserFile to collection"""
            col._collection.append(itm)

        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)
        mock_ufile.return_value = user_file_gen("1")
        FileCollection.add = add
        mock_api.query_files.return_value = resp
        mock_mess.return_value = ["1", "2", "3", "4", "5"]

        col = FileCollection(mock_api)
        upl = col.is_uploaded()
        self.assertIsInstance(upl, FileCollection)
        self.assertEqual(upl._collection, col._collection)
        self.assertFalse(mock_api.query_files.called)

        col._collection = [1, 2, 3, 4, 5]
        with self.assertRaises(RestCallException):
            col.is_uploaded()
        mock_api.query_files.assert_called_once_with(["1", "2", "3", "4", "5"])

        with self.assertRaises(RestCallException):
            col.is_uploaded(per_call=2)
        mock_api.query_files.assert_called_with(["1", "2"])

        col._collection = [user_file_gen("1"), user_file_gen("2")]
        mock_api.reset()
        resp.success = True
        resp.result = ["test1", "test2", "test3"]
        upl = col.is_uploaded()
        mock_api.query_files.assert_called_with(["1", "2", "3", "4", "5"])
        mock_rem.assert_called_with([mock.ANY])
        self.assertEqual(upl._collection, col._collection)

        col._collection = [user_file_gen("test1"), user_file_gen("test2")]
        upl = col.is_uploaded()
        mock_rem.assert_called_with([])
        self.assertEqual(upl._collection, col._collection)
    def test_filecoll_iter(self, mock_api, mock_add):
        """Test __iter__"""

        col = FileCollection(mock_api)

        itr = iter(col)
        with self.assertRaises(StopIteration):
            next(itr)

        col._collection.append(None)
        for ufile in col:
            self.assertIsNone(ufile)
    def test_filecoll_upload_thread(self, mock_pik, mock_api):
        """Test upload"""

        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)

        col = FileCollection(mock_api)
        col._api = None
        failed = col.upload(force=True, threads=1)
        self.assertFalse(mock_pik.called)
        self.assertEqual(failed, [])

        col._collection = [1, 2, 3, 4]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, 
                         [(1, "'int' object has no attribute 'upload'"),
                          (2, "'int' object has no attribute 'upload'"),
                          (3, "'int' object has no attribute 'upload'"),
                          (4, "'int' object has no attribute 'upload'")])

        mock_pik.call_count = 0
        col._collection = [UFile()]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(len(failed), 1)
        self.assertIsInstance(failed[0], tuple)

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True)]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, [])

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True)]
        failed = col.upload(force=True, threads=3)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, [])

        mock_pik.call_count = 0
        col._collection = [UFile() for a in range(15)]
        failed = col.upload(force=True, threads=3)
        self.assertEqual(mock_pik.call_count, 5)
        self.assertEqual(len(failed), 15)

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True) for a in range(20)]
        failed = col.upload(force=True, threads=20)
        self.assertEqual(mock_pik.call_count, 2)
        self.assertEqual(failed, [])
Exemple #10
0
    def add_file(self, userfile):
        """
        Add userfile to required files list.
        If the job does not have an :class:`.FileCollection` already assigned,
        a new one will be created.

        :Args:
            - userfile (:class:`.UserFile`): The userfile to be added to
              the job.
        """
        if self.required_files is None:
            self.required_files = FileCollection(self._api)

        self.required_files.add(userfile)
    def test_filecoll_index(self, mock_api):
        """Test index"""

        col = FileCollection(mock_api)

        test_file = mock.create_autospec(UserFile)
        test_file2 = mock.create_autospec(UserFile)
        test_file3 = mock.create_autospec(UserFile)

        col._collection = [test_file, test_file2]

        with self.assertRaises(TypeError):
            col.index(None)
        with self.assertRaises(TypeError):
            col.index("test")

        with self.assertRaises(ValueError):
            col.index(test_file3)

        self.assertEqual(col._collection.index(test_file2), 1)
    def test_filecoll_upload(self, mock_isup, mock_upload, mock_api):
        """Test upload"""

        _callback = mock.Mock()
        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)
        mock_isup.return_value = []
        mock_upload.return_value = (False, "f", "Error!")

        mock_isup.called = False
        col = FileCollection(mock_api)
        failed = col.upload()
        self.assertTrue(mock_isup.called)
        self.assertFalse(mock_upload.called)
        self.assertEqual(failed, [])

        mock_isup.called = False
        failed = col.upload(force=True)
        self.assertFalse(mock_isup.called)
        self.assertFalse(mock_upload.called)
        self.assertEqual(failed, [])

        col._collection = [1, 2, 3, 4]
        failed = col.upload(force=True)
        mock_upload.assert_any_call(1, callback=None, block=4096)
        self.assertEqual(mock_upload.call_count, 4)

        self.assertEqual(failed, [("f", "Error!"),
                                  ("f", "Error!"),
                                  ("f", "Error!"),
                                  ("f", "Error!")])

        mock_upload.call_count = 0
        resp.success = True
        mock_upload.return_value = (True, "f", "All good!")
        failed = col.upload(force=True, threads=None, callback=_callback, block=1)
        mock_upload.assert_any_call(1, callback=_callback, block=1)
        self.assertEqual(mock_upload.call_count, 4)
        self.assertEqual(failed, [])
    def test_filecoll_get_message(self, mock_api, mock_add):
        """Test _get_message"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)
        test_file.create_query_specifier.return_value = {"test_query":1}
        test_file.create_submit_specifier.return_value = {"test_submit":2}
        col._collection = [test_file]

        specs = col._get_message(None)
        self.assertEqual(specs, [])

        specs = col._get_message(1)
        self.assertEqual(specs, [])

        specs = col._get_message([])
        self.assertEqual(specs, [])

        specs = col._get_message("query")
        self.assertEqual(specs, [{"test_query":1}])
        specs = col._get_message("submit")
        self.assertEqual(specs, [{"test_submit":2}])
    def test_filecoll_delitem(self, mock_api, mock_add):
        """Test __delitem__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        del col[0]
        del col[-1]
        del col[1:]
        self.assertEqual(col._collection, [None])

        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"
        col._collection = [test_file]

        del col["test"]
        self.assertEqual(col._collection, [])
        del col["something"]
        self.assertEqual(col._collection, [])

        del col[5]
        del col[None]
        del col[0:-1]
    def test_filecoll_getitem(self, mock_api, mock_add):
        """Test __getitem__"""

        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"

        col = FileCollection(mock_api)
        with self.assertRaises(FileMissingException):
            print(col[1])

        self.assertEqual(col[:1], [])
        col._collection.append(test_file)

        self.assertEqual(col[0], test_file)
        self.assertEqual(col["test"], [test_file])
        self.assertEqual(col[-1], test_file)
        self.assertEqual(col[:1], [test_file])

        with self.assertRaises(FileMissingException):
            print(col[10])
        with self.assertRaises(FileMissingException):
            print(col["test2"])
        with self.assertRaises(FileMissingException):
            print(col[None])
    def test_filecoll_create(self, mock_api, mock_add):
        """Test FileCollection object"""

        with self.assertRaises(TypeError):
            FileCollection(None)

        FileCollection(mock_api)
        self.assertFalse(mock_add.called)

        FileCollection(mock_api, "test")
        mock_add.assert_called_once_with("test")

        FileCollection(mock_api, "test1", "test2")
        mock_add.assert_any_call("test1")
        mock_add.assert_any_call("test2")

        FileCollection(mock_api, ["test1", "test2"])
        mock_add.assert_any_call("test1")
        mock_add.assert_any_call("test2")

        FileCollection(mock_api, None)
        mock_add.assert_any_call(None)