def test_filecoll_extend(self, mock_api): """Test extend""" col = FileCollection(mock_api) col2 = FileCollection(mock_api) test_file = mock.create_autospec(UserFile) test_file2 = mock.create_autospec(UserFile) col._collection = [test_file] col2._collection = [test_file2, test_file] with self.assertRaises(AttributeError): col.extend(None) with self.assertRaises(AttributeError): col.extend("test") with self.assertRaises(AttributeError): col.extend([]) col.extend(col2) self.assertEqual(len(col._collection), 2) self.assertTrue(all(i in [test_file, test_file2] for i in col._collection)) col2.extend(col) self.assertEqual(len(col._collection), 2) self.assertTrue(all(i in [test_file, test_file2] for i in col._collection))
def test_filecoll_remove(self, mock_api): """Test remove""" col = FileCollection(mock_api) test_file = mock.create_autospec(UserFile) test_file.name = "test" col._collection = [test_file, 1, "2", None, []] with self.assertRaises(TypeError): col.remove(None) with self.assertRaises(TypeError): col.remove(10) col.remove(1) col.remove(-1) col.remove(slice(1)) self.assertEqual(col._collection, ["2", None]) test_file2 = mock.create_autospec(UserFile) test_file2.name = "test2" test_file3 = mock.create_autospec(UserFile) test_file3.name = "test3" col._collection = [test_file, test_file2, test_file3] col.remove("test") self.assertEqual(col._collection, [test_file2, test_file3]) col.remove(["test2", "test3"]) self.assertEqual(col._collection, [])
def add_file_collection(self, file_collection=None): """ Assign a file collection to the job. The userfiles assigned to a job submission will be loaded onto each node prior to the job being executed. :Kwargs: - file_collection (:py:class:`.FileCollection`): If set, this will be assigned as the :py:class:`.FileCollection` of the job. If not set, a new :py:class:`.FileCollection` will be created. :Raises: - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is passed in. """ if file_collection is None: self._log.info("Assigning empty FileCollection to job") self.required_files = FileCollection(self._api) elif hasattr(file_collection, 'add'): self._log.debug("Assigning FileCollection with {0} " "userfiles to job".format(len(file_collection))) self.required_files = file_collection else: raise TypeError("Can only assign an object of type FileCollection" ", not {type}".format(type=type(file_collection)))
def test_filecoll_add(self, mock_api): """Test add""" col = FileCollection(mock_api) test_file = mock.create_autospec(UserFile) with self.assertRaises(FileInvalidException): col.add("test") with self.assertRaises(FileInvalidException): col.add(1) with self.assertRaises(FileInvalidException): col.add(None) col.add(test_file) self.assertEqual(col._collection, [test_file]) with self.assertRaises(FileInvalidException): col.add(test_file) col._collection = [] col.add([test_file]) self.assertEqual(col._collection, [test_file]) col.add([test_file]) self.assertEqual(col._collection, [test_file]) col._collection = [] col.add([test_file, test_file]) col._collection = [] col.add([1, "2", None]) self.assertEqual(col._collection, [])
def test_filecoll_str(self, mock_api, mock_add): """Test __str__""" col = FileCollection(mock_api) col._collection = [1, None, "test", [], {}] colstr = str(col) self.assertEqual(colstr, "['1', 'None', 'test', '[]', '{}']")
def test_filecoll_len(self, mock_api, mock_add): """Test __len__""" col = FileCollection(mock_api) col._collection = [1, None, "test", [], {}] self.assertEqual(len(col), len(col._collection)) col._collection.append("more") self.assertEqual(len(col), len(col._collection))
def test_filecoll_is_uploaded(self, mock_rem, mock_mess, mock_ufile, mock_api): """Test is_uploaded""" def user_file_gen(u_name): """Mock UserFile generator""" ugen = mock.create_autospec(UserFile) ugen.name = str(u_name) ugen.compare_lastmodified.return_value = True return ugen def add(col, itm): """Mock add UserFile to collection""" col._collection.append(itm) resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "Boom", None) mock_ufile.return_value = user_file_gen("1") FileCollection.add = add mock_api.query_files.return_value = resp mock_mess.return_value = ["1", "2", "3", "4", "5"] col = FileCollection(mock_api) upl = col.is_uploaded() self.assertIsInstance(upl, FileCollection) self.assertEqual(upl._collection, col._collection) self.assertFalse(mock_api.query_files.called) col._collection = [1, 2, 3, 4, 5] with self.assertRaises(RestCallException): col.is_uploaded() mock_api.query_files.assert_called_once_with(["1", "2", "3", "4", "5"]) with self.assertRaises(RestCallException): col.is_uploaded(per_call=2) mock_api.query_files.assert_called_with(["1", "2"]) col._collection = [user_file_gen("1"), user_file_gen("2")] mock_api.reset() resp.success = True resp.result = ["test1", "test2", "test3"] upl = col.is_uploaded() mock_api.query_files.assert_called_with(["1", "2", "3", "4", "5"]) mock_rem.assert_called_with([mock.ANY]) self.assertEqual(upl._collection, col._collection) col._collection = [user_file_gen("test1"), user_file_gen("test2")] upl = col.is_uploaded() mock_rem.assert_called_with([]) self.assertEqual(upl._collection, col._collection)
def test_filecoll_iter(self, mock_api, mock_add): """Test __iter__""" col = FileCollection(mock_api) itr = iter(col) with self.assertRaises(StopIteration): next(itr) col._collection.append(None) for ufile in col: self.assertIsNone(ufile)
def test_filecoll_upload_thread(self, mock_pik, mock_api): """Test upload""" resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "Boom", None) col = FileCollection(mock_api) col._api = None failed = col.upload(force=True, threads=1) self.assertFalse(mock_pik.called) self.assertEqual(failed, []) col._collection = [1, 2, 3, 4] failed = col.upload(force=True, threads=1) self.assertEqual(mock_pik.call_count, 1) self.assertEqual(failed, [(1, "'int' object has no attribute 'upload'"), (2, "'int' object has no attribute 'upload'"), (3, "'int' object has no attribute 'upload'"), (4, "'int' object has no attribute 'upload'")]) mock_pik.call_count = 0 col._collection = [UFile()] failed = col.upload(force=True, threads=1) self.assertEqual(mock_pik.call_count, 1) self.assertEqual(len(failed), 1) self.assertIsInstance(failed[0], tuple) mock_pik.call_count = 0 col._collection = [UFile(arg_a=True)] failed = col.upload(force=True, threads=1) self.assertEqual(mock_pik.call_count, 1) self.assertEqual(failed, []) mock_pik.call_count = 0 col._collection = [UFile(arg_a=True)] failed = col.upload(force=True, threads=3) self.assertEqual(mock_pik.call_count, 1) self.assertEqual(failed, []) mock_pik.call_count = 0 col._collection = [UFile() for a in range(15)] failed = col.upload(force=True, threads=3) self.assertEqual(mock_pik.call_count, 5) self.assertEqual(len(failed), 15) mock_pik.call_count = 0 col._collection = [UFile(arg_a=True) for a in range(20)] failed = col.upload(force=True, threads=20) self.assertEqual(mock_pik.call_count, 2) self.assertEqual(failed, [])
def add_file(self, userfile): """ Add userfile to required files list. If the job does not have an :class:`.FileCollection` already assigned, a new one will be created. :Args: - userfile (:class:`.UserFile`): The userfile to be added to the job. """ if self.required_files is None: self.required_files = FileCollection(self._api) self.required_files.add(userfile)
def test_filecoll_index(self, mock_api): """Test index""" col = FileCollection(mock_api) test_file = mock.create_autospec(UserFile) test_file2 = mock.create_autospec(UserFile) test_file3 = mock.create_autospec(UserFile) col._collection = [test_file, test_file2] with self.assertRaises(TypeError): col.index(None) with self.assertRaises(TypeError): col.index("test") with self.assertRaises(ValueError): col.index(test_file3) self.assertEqual(col._collection.index(test_file2), 1)
def test_filecoll_upload(self, mock_isup, mock_upload, mock_api): """Test upload""" _callback = mock.Mock() resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "Boom", None) mock_isup.return_value = [] mock_upload.return_value = (False, "f", "Error!") mock_isup.called = False col = FileCollection(mock_api) failed = col.upload() self.assertTrue(mock_isup.called) self.assertFalse(mock_upload.called) self.assertEqual(failed, []) mock_isup.called = False failed = col.upload(force=True) self.assertFalse(mock_isup.called) self.assertFalse(mock_upload.called) self.assertEqual(failed, []) col._collection = [1, 2, 3, 4] failed = col.upload(force=True) mock_upload.assert_any_call(1, callback=None, block=4096) self.assertEqual(mock_upload.call_count, 4) self.assertEqual(failed, [("f", "Error!"), ("f", "Error!"), ("f", "Error!"), ("f", "Error!")]) mock_upload.call_count = 0 resp.success = True mock_upload.return_value = (True, "f", "All good!") failed = col.upload(force=True, threads=None, callback=_callback, block=1) mock_upload.assert_any_call(1, callback=_callback, block=1) self.assertEqual(mock_upload.call_count, 4) self.assertEqual(failed, [])
def test_filecoll_get_message(self, mock_api, mock_add): """Test _get_message""" col = FileCollection(mock_api) test_file = mock.create_autospec(UserFile) test_file.create_query_specifier.return_value = {"test_query":1} test_file.create_submit_specifier.return_value = {"test_submit":2} col._collection = [test_file] specs = col._get_message(None) self.assertEqual(specs, []) specs = col._get_message(1) self.assertEqual(specs, []) specs = col._get_message([]) self.assertEqual(specs, []) specs = col._get_message("query") self.assertEqual(specs, [{"test_query":1}]) specs = col._get_message("submit") self.assertEqual(specs, [{"test_submit":2}])
def test_filecoll_delitem(self, mock_api, mock_add): """Test __delitem__""" col = FileCollection(mock_api) col._collection = [1, None, "test", [], {}] del col[0] del col[-1] del col[1:] self.assertEqual(col._collection, [None]) test_file = mock.create_autospec(UserFile) test_file.name = "test" col._collection = [test_file] del col["test"] self.assertEqual(col._collection, []) del col["something"] self.assertEqual(col._collection, []) del col[5] del col[None] del col[0:-1]
def test_filecoll_getitem(self, mock_api, mock_add): """Test __getitem__""" test_file = mock.create_autospec(UserFile) test_file.name = "test" col = FileCollection(mock_api) with self.assertRaises(FileMissingException): print(col[1]) self.assertEqual(col[:1], []) col._collection.append(test_file) self.assertEqual(col[0], test_file) self.assertEqual(col["test"], [test_file]) self.assertEqual(col[-1], test_file) self.assertEqual(col[:1], [test_file]) with self.assertRaises(FileMissingException): print(col[10]) with self.assertRaises(FileMissingException): print(col["test2"]) with self.assertRaises(FileMissingException): print(col[None])
def test_filecoll_create(self, mock_api, mock_add): """Test FileCollection object""" with self.assertRaises(TypeError): FileCollection(None) FileCollection(mock_api) self.assertFalse(mock_add.called) FileCollection(mock_api, "test") mock_add.assert_called_once_with("test") FileCollection(mock_api, "test1", "test2") mock_add.assert_any_call("test1") mock_add.assert_any_call("test2") FileCollection(mock_api, ["test1", "test2"]) mock_add.assert_any_call("test1") mock_add.assert_any_call("test2") FileCollection(mock_api, None) mock_add.assert_any_call(None)