def test_filecoll_extend(self, mock_api):
"""Test extend"""
col = FileCollection(mock_api)
col2 = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file2 = mock.create_autospec(UserFile)
col._collection = [test_file]
col2._collection = [test_file2, test_file]
with self.assertRaises(AttributeError):
col.extend(None)
with self.assertRaises(AttributeError):
col.extend("test")
with self.assertRaises(AttributeError):
col.extend([])
col.extend(col2)
self.assertEqual(len(col._collection), 2)
self.assertTrue(all(i in [test_file, test_file2]
for i in col._collection))
col2.extend(col)
self.assertEqual(len(col._collection), 2)
self.assertTrue(all(i in [test_file, test_file2]
for i in col._collection))
def test_filecoll_remove(self, mock_api):
"""Test remove"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col._collection = [test_file, 1, "2", None, []]
with self.assertRaises(TypeError):
col.remove(None)
with self.assertRaises(TypeError):
col.remove(10)
col.remove(1)
col.remove(-1)
col.remove(slice(1))
self.assertEqual(col._collection, ["2", None])
test_file2 = mock.create_autospec(UserFile)
test_file2.name = "test2"
test_file3 = mock.create_autospec(UserFile)
test_file3.name = "test3"
col._collection = [test_file, test_file2, test_file3]
col.remove("test")
self.assertEqual(col._collection, [test_file2, test_file3])
col.remove(["test2", "test3"])
self.assertEqual(col._collection, [])
def add_file_collection(self, file_collection=None):
"""
Assign a file collection to the job.
The userfiles assigned to a job submission will be loaded onto each
node prior to the job being executed.
:Kwargs:
- file_collection (:py:class:`.FileCollection`): If set, this will
be assigned as the :py:class:`.FileCollection` of the job. If
not set, a new :py:class:`.FileCollection` will be created.
:Raises:
- :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
passed in.
"""
if file_collection is None:
self._log.info("Assigning empty FileCollection to job")
self.required_files = FileCollection(self._api)
elif hasattr(file_collection, 'add'):
self._log.debug("Assigning FileCollection with {0} "
"userfiles to job".format(len(file_collection)))
self.required_files = file_collection
else:
raise TypeError("Can only assign an object of type FileCollection"
", not {type}".format(type=type(file_collection)))
def test_filecoll_add(self, mock_api):
"""Test add"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
with self.assertRaises(FileInvalidException):
col.add("test")
with self.assertRaises(FileInvalidException):
col.add(1)
with self.assertRaises(FileInvalidException):
col.add(None)
col.add(test_file)
self.assertEqual(col._collection, [test_file])
with self.assertRaises(FileInvalidException):
col.add(test_file)
col._collection = []
col.add([test_file])
self.assertEqual(col._collection, [test_file])
col.add([test_file])
self.assertEqual(col._collection, [test_file])
col._collection = []
col.add([test_file, test_file])
col._collection = []
col.add([1, "2", None])
self.assertEqual(col._collection, [])
def test_filecoll_str(self, mock_api, mock_add):
"""Test __str__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
colstr = str(col)
self.assertEqual(colstr, "['1', 'None', 'test', '[]', '{}']")
def test_filecoll_len(self, mock_api, mock_add):
"""Test __len__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
self.assertEqual(len(col), len(col._collection))
col._collection.append("more")
self.assertEqual(len(col), len(col._collection))
def test_filecoll_is_uploaded(self,
mock_rem,
mock_mess,
mock_ufile,
mock_api):
"""Test is_uploaded"""
def user_file_gen(u_name):
"""Mock UserFile generator"""
ugen = mock.create_autospec(UserFile)
ugen.name = str(u_name)
ugen.compare_lastmodified.return_value = True
return ugen
def add(col, itm):
"""Mock add UserFile to collection"""
col._collection.append(itm)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
mock_ufile.return_value = user_file_gen("1")
FileCollection.add = add
mock_api.query_files.return_value = resp
mock_mess.return_value = ["1", "2", "3", "4", "5"]
col = FileCollection(mock_api)
upl = col.is_uploaded()
self.assertIsInstance(upl, FileCollection)
self.assertEqual(upl._collection, col._collection)
self.assertFalse(mock_api.query_files.called)
col._collection = [1, 2, 3, 4, 5]
with self.assertRaises(RestCallException):
col.is_uploaded()
mock_api.query_files.assert_called_once_with(["1", "2", "3", "4", "5"])
with self.assertRaises(RestCallException):
col.is_uploaded(per_call=2)
mock_api.query_files.assert_called_with(["1", "2"])
col._collection = [user_file_gen("1"), user_file_gen("2")]
mock_api.reset()
resp.success = True
resp.result = ["test1", "test2", "test3"]
upl = col.is_uploaded()
mock_api.query_files.assert_called_with(["1", "2", "3", "4", "5"])
mock_rem.assert_called_with([mock.ANY])
self.assertEqual(upl._collection, col._collection)
col._collection = [user_file_gen("test1"), user_file_gen("test2")]
upl = col.is_uploaded()
mock_rem.assert_called_with([])
self.assertEqual(upl._collection, col._collection)
def test_filecoll_iter(self, mock_api, mock_add):
"""Test __iter__"""
col = FileCollection(mock_api)
itr = iter(col)
with self.assertRaises(StopIteration):
next(itr)
col._collection.append(None)
for ufile in col:
self.assertIsNone(ufile)
def test_filecoll_upload_thread(self, mock_pik, mock_api):
"""Test upload"""
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
col = FileCollection(mock_api)
col._api = None
failed = col.upload(force=True, threads=1)
self.assertFalse(mock_pik.called)
self.assertEqual(failed, [])
col._collection = [1, 2, 3, 4]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed,
[(1, "'int' object has no attribute 'upload'"),
(2, "'int' object has no attribute 'upload'"),
(3, "'int' object has no attribute 'upload'"),
(4, "'int' object has no attribute 'upload'")])
mock_pik.call_count = 0
col._collection = [UFile()]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(len(failed), 1)
self.assertIsInstance(failed[0], tuple)
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True)]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed, [])
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True)]
failed = col.upload(force=True, threads=3)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed, [])
mock_pik.call_count = 0
col._collection = [UFile() for a in range(15)]
failed = col.upload(force=True, threads=3)
self.assertEqual(mock_pik.call_count, 5)
self.assertEqual(len(failed), 15)
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True) for a in range(20)]
failed = col.upload(force=True, threads=20)
self.assertEqual(mock_pik.call_count, 2)
self.assertEqual(failed, [])
def add_file(self, userfile):
"""
Add userfile to required files list.
If the job does not have an :class:`.FileCollection` already assigned,
a new one will be created.
:Args:
- userfile (:class:`.UserFile`): The userfile to be added to
the job.
"""
if self.required_files is None:
self.required_files = FileCollection(self._api)
self.required_files.add(userfile)
def test_filecoll_index(self, mock_api):
"""Test index"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file2 = mock.create_autospec(UserFile)
test_file3 = mock.create_autospec(UserFile)
col._collection = [test_file, test_file2]
with self.assertRaises(TypeError):
col.index(None)
with self.assertRaises(TypeError):
col.index("test")
with self.assertRaises(ValueError):
col.index(test_file3)
self.assertEqual(col._collection.index(test_file2), 1)
def test_filecoll_upload(self, mock_isup, mock_upload, mock_api):
"""Test upload"""
_callback = mock.Mock()
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
mock_isup.return_value = []
mock_upload.return_value = (False, "f", "Error!")
mock_isup.called = False
col = FileCollection(mock_api)
failed = col.upload()
self.assertTrue(mock_isup.called)
self.assertFalse(mock_upload.called)
self.assertEqual(failed, [])
mock_isup.called = False
failed = col.upload(force=True)
self.assertFalse(mock_isup.called)
self.assertFalse(mock_upload.called)
self.assertEqual(failed, [])
col._collection = [1, 2, 3, 4]
failed = col.upload(force=True)
mock_upload.assert_any_call(1, callback=None, block=4096)
self.assertEqual(mock_upload.call_count, 4)
self.assertEqual(failed, [("f", "Error!"),
("f", "Error!"),
("f", "Error!"),
("f", "Error!")])
mock_upload.call_count = 0
resp.success = True
mock_upload.return_value = (True, "f", "All good!")
failed = col.upload(force=True, threads=None, callback=_callback, block=1)
mock_upload.assert_any_call(1, callback=_callback, block=1)
self.assertEqual(mock_upload.call_count, 4)
self.assertEqual(failed, [])
def test_filecoll_get_message(self, mock_api, mock_add):
"""Test _get_message"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file.create_query_specifier.return_value = {"test_query":1}
test_file.create_submit_specifier.return_value = {"test_submit":2}
col._collection = [test_file]
specs = col._get_message(None)
self.assertEqual(specs, [])
specs = col._get_message(1)
self.assertEqual(specs, [])
specs = col._get_message([])
self.assertEqual(specs, [])
specs = col._get_message("query")
self.assertEqual(specs, [{"test_query":1}])
specs = col._get_message("submit")
self.assertEqual(specs, [{"test_submit":2}])
def test_filecoll_delitem(self, mock_api, mock_add):
"""Test __delitem__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
del col[0]
del col[-1]
del col[1:]
self.assertEqual(col._collection, [None])
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col._collection = [test_file]
del col["test"]
self.assertEqual(col._collection, [])
del col["something"]
self.assertEqual(col._collection, [])
del col[5]
del col[None]
del col[0:-1]
def test_filecoll_getitem(self, mock_api, mock_add):
"""Test __getitem__"""
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col = FileCollection(mock_api)
with self.assertRaises(FileMissingException):
print(col[1])
self.assertEqual(col[:1], [])
col._collection.append(test_file)
self.assertEqual(col[0], test_file)
self.assertEqual(col["test"], [test_file])
self.assertEqual(col[-1], test_file)
self.assertEqual(col[:1], [test_file])
with self.assertRaises(FileMissingException):
print(col[10])
with self.assertRaises(FileMissingException):
print(col["test2"])
with self.assertRaises(FileMissingException):
print(col[None])
def test_filecoll_create(self, mock_api, mock_add):
"""Test FileCollection object"""
with self.assertRaises(TypeError):
FileCollection(None)
FileCollection(mock_api)
self.assertFalse(mock_add.called)
FileCollection(mock_api, "test")
mock_add.assert_called_once_with("test")
FileCollection(mock_api, "test1", "test2")
mock_add.assert_any_call("test1")
mock_add.assert_any_call("test2")
FileCollection(mock_api, ["test1", "test2"])
mock_add.assert_any_call("test1")
mock_add.assert_any_call("test2")
FileCollection(mock_api, None)
mock_add.assert_any_call(None)
