def add_file_collection(self, file_collection=None):
"""
Assign a file collection to the job.
The userfiles assigned to a job submission will be loaded onto each
node prior to the job being executed.
:Kwargs:
- file_collection (:py:class:`.FileCollection`): If set, this will
be assigned as the :py:class:`.FileCollection` of the job. If
not set, a new :py:class:`.FileCollection` will be created.
:Raises:
- :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
passed in.
"""
if file_collection is None:
self._log.info("Assigning empty FileCollection to job")
self.required_files = FileCollection(self._api)
elif hasattr(file_collection, 'add'):
self._log.debug("Assigning FileCollection with {0} "
"userfiles to job".format(len(file_collection)))
self.required_files = file_collection
else:
raise TypeError("Can only assign an object of type FileCollection"
", not {type}".format(type=type(file_collection)))
def test_filecoll_str(self, mock_api, mock_add):
"""Test __str__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
colstr = str(col)
self.assertEqual(colstr, "['1', 'None', 'test', '[]', '{}']")
def test_filecoll_len(self, mock_api, mock_add):
"""Test __len__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
self.assertEqual(len(col), len(col._collection))
col._collection.append("more")
self.assertEqual(len(col), len(col._collection))
def add_file(self, userfile):
"""
Add userfile to required files list.
If the job does not have an :class:`.FileCollection` already assigned,
a new one will be created.
:Args:
- userfile (:class:`.UserFile`): The userfile to be added to
the job.
"""
if self.required_files is None:
self.required_files = FileCollection(self._api)
self.required_files.add(userfile)
def add_file_collection(self, file_collection=None):
"""
Assign a file collection to the job.
The userfiles assigned to a job submission will be loaded onto each
node prior to the job being executed.
:Kwargs:
- file_collection (:py:class:`.FileCollection`): If set, this will
be assigned as the :py:class:`.FileCollection` of the job. If
not set, a new :py:class:`.FileCollection` will be created.
:Raises:
- :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
passed in.
"""
if file_collection is None:
self._log.info("Assigning empty FileCollection to job")
self.required_files = FileCollection(self._api)
elif hasattr(file_collection, 'add'):
self._log.debug("Assigning FileCollection with {0} "
"userfiles to job".format(len(file_collection)))
self.required_files = file_collection
else:
raise TypeError("Can only assign an object of type FileCollection"
", not {type}".format(type=type(file_collection)))
def test_filecoll_iter(self, mock_api, mock_add):
"""Test __iter__"""
col = FileCollection(mock_api)
itr = iter(col)
with self.assertRaises(StopIteration):
next(itr)
col._collection.append(None)
for ufile in col:
self.assertIsNone(ufile)
def test_filecoll_delitem(self, mock_api, mock_add):
"""Test __delitem__"""
col = FileCollection(mock_api)
col._collection = [1, None, "test", [], {}]
del col[0]
del col[-1]
del col[1:]
self.assertEqual(col._collection, [None])
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col._collection = [test_file]
del col["test"]
self.assertEqual(col._collection, [])
del col["something"]
self.assertEqual(col._collection, [])
del col[5]
del col[None]
del col[0:-1]
def add_file(self, userfile):
"""
Add userfile to required files list.
If the job does not have an :class:`.FileCollection` already assigned,
a new one will be created.
:Args:
- userfile (:class:`.UserFile`): The userfile to be added to
the job.
"""
if self.required_files is None:
self.required_files = FileCollection(self._api)
self.required_files.add(userfile)
def test_filecoll_index(self, mock_api):
"""Test index"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file2 = mock.create_autospec(UserFile)
test_file3 = mock.create_autospec(UserFile)
col._collection = [test_file, test_file2]
with self.assertRaises(TypeError):
col.index(None)
with self.assertRaises(TypeError):
col.index("test")
with self.assertRaises(ValueError):
col.index(test_file3)
self.assertEqual(col._collection.index(test_file2), 1)
def test_filecoll_get_message(self, mock_api, mock_add):
"""Test _get_message"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file.create_query_specifier.return_value = {"test_query":1}
test_file.create_submit_specifier.return_value = {"test_submit":2}
col._collection = [test_file]
specs = col._get_message(None)
self.assertEqual(specs, [])
specs = col._get_message(1)
self.assertEqual(specs, [])
specs = col._get_message([])
self.assertEqual(specs, [])
specs = col._get_message("query")
self.assertEqual(specs, [{"test_query":1}])
specs = col._get_message("submit")
self.assertEqual(specs, [{"test_submit":2}])
def test_filecoll_upload(self, mock_isup, mock_upload, mock_api):
"""Test upload"""
_callback = mock.Mock()
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
mock_isup.return_value = []
mock_upload.return_value = (False, "f", "Error!")
mock_isup.called = False
col = FileCollection(mock_api)
failed = col.upload()
self.assertTrue(mock_isup.called)
self.assertFalse(mock_upload.called)
self.assertEqual(failed, [])
mock_isup.called = False
failed = col.upload(force=True)
self.assertFalse(mock_isup.called)
self.assertFalse(mock_upload.called)
self.assertEqual(failed, [])
col._collection = [1, 2, 3, 4]
failed = col.upload(force=True)
mock_upload.assert_any_call(1, callback=None, block=4096)
self.assertEqual(mock_upload.call_count, 4)
self.assertEqual(failed, [("f", "Error!"),
("f", "Error!"),
("f", "Error!"),
("f", "Error!")])
mock_upload.call_count = 0
resp.success = True
mock_upload.return_value = (True, "f", "All good!")
failed = col.upload(force=True, threads=None, callback=_callback, block=1)
mock_upload.assert_any_call(1, callback=_callback, block=1)
self.assertEqual(mock_upload.call_count, 4)
self.assertEqual(failed, [])
def test_filecoll_getitem(self, mock_api, mock_add):
"""Test __getitem__"""
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col = FileCollection(mock_api)
with self.assertRaises(FileMissingException):
print(col[1])
self.assertEqual(col[:1], [])
col._collection.append(test_file)
self.assertEqual(col[0], test_file)
self.assertEqual(col["test"], [test_file])
self.assertEqual(col[-1], test_file)
self.assertEqual(col[:1], [test_file])
with self.assertRaises(FileMissingException):
print(col[10])
with self.assertRaises(FileMissingException):
print(col["test2"])
with self.assertRaises(FileMissingException):
print(col[None])
def test_filecoll_create(self, mock_api, mock_add):
"""Test FileCollection object"""
with self.assertRaises(TypeError):
FileCollection(None)
FileCollection(mock_api)
self.assertFalse(mock_add.called)
FileCollection(mock_api, "test")
mock_add.assert_called_once_with("test")
FileCollection(mock_api, "test1", "test2")
mock_add.assert_any_call("test1")
mock_add.assert_any_call("test2")
FileCollection(mock_api, ["test1", "test2"])
mock_add.assert_any_call("test1")
mock_add.assert_any_call("test2")
FileCollection(mock_api, None)
mock_add.assert_any_call(None)
def test_filecoll_upload_thread(self, mock_pik, mock_api):
"""Test upload"""
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
col = FileCollection(mock_api)
col._api = None
failed = col.upload(force=True, threads=1)
self.assertFalse(mock_pik.called)
self.assertEqual(failed, [])
col._collection = [1, 2, 3, 4]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed,
[(1, "'int' object has no attribute 'upload'"),
(2, "'int' object has no attribute 'upload'"),
(3, "'int' object has no attribute 'upload'"),
(4, "'int' object has no attribute 'upload'")])
mock_pik.call_count = 0
col._collection = [UFile()]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(len(failed), 1)
self.assertIsInstance(failed[0], tuple)
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True)]
failed = col.upload(force=True, threads=1)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed, [])
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True)]
failed = col.upload(force=True, threads=3)
self.assertEqual(mock_pik.call_count, 1)
self.assertEqual(failed, [])
mock_pik.call_count = 0
col._collection = [UFile() for a in range(15)]
failed = col.upload(force=True, threads=3)
self.assertEqual(mock_pik.call_count, 5)
self.assertEqual(len(failed), 15)
mock_pik.call_count = 0
col._collection = [UFile(arg_a=True) for a in range(20)]
failed = col.upload(force=True, threads=20)
self.assertEqual(mock_pik.call_count, 2)
self.assertEqual(failed, [])
def test_filecoll_is_uploaded(self,
mock_rem,
mock_mess,
mock_ufile,
mock_api):
"""Test is_uploaded"""
def user_file_gen(u_name):
"""Mock UserFile generator"""
ugen = mock.create_autospec(UserFile)
ugen.name = str(u_name)
ugen.compare_lastmodified.return_value = True
return ugen
def add(col, itm):
"""Mock add UserFile to collection"""
col._collection.append(itm)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "Boom", None)
mock_ufile.return_value = user_file_gen("1")
FileCollection.add = add
mock_api.query_files.return_value = resp
mock_mess.return_value = ["1", "2", "3", "4", "5"]
col = FileCollection(mock_api)
upl = col.is_uploaded()
self.assertIsInstance(upl, FileCollection)
self.assertEqual(upl._collection, col._collection)
self.assertFalse(mock_api.query_files.called)
col._collection = [1, 2, 3, 4, 5]
with self.assertRaises(RestCallException):
col.is_uploaded()
mock_api.query_files.assert_called_once_with(["1", "2", "3", "4", "5"])
with self.assertRaises(RestCallException):
col.is_uploaded(per_call=2)
mock_api.query_files.assert_called_with(["1", "2"])
col._collection = [user_file_gen("1"), user_file_gen("2")]
mock_api.reset()
resp.success = True
resp.result = ["test1", "test2", "test3"]
upl = col.is_uploaded()
mock_api.query_files.assert_called_with(["1", "2", "3", "4", "5"])
mock_rem.assert_called_with([mock.ANY])
self.assertEqual(upl._collection, col._collection)
col._collection = [user_file_gen("test1"), user_file_gen("test2")]
upl = col.is_uploaded()
mock_rem.assert_called_with([])
self.assertEqual(upl._collection, col._collection)
def test_filecoll_remove(self, mock_api):
"""Test remove"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file.name = "test"
col._collection = [test_file, 1, "2", None, []]
with self.assertRaises(TypeError):
col.remove(None)
with self.assertRaises(TypeError):
col.remove(10)
col.remove(1)
col.remove(-1)
col.remove(slice(1))
self.assertEqual(col._collection, ["2", None])
test_file2 = mock.create_autospec(UserFile)
test_file2.name = "test2"
test_file3 = mock.create_autospec(UserFile)
test_file3.name = "test3"
col._collection = [test_file, test_file2, test_file3]
col.remove("test")
self.assertEqual(col._collection, [test_file2, test_file3])
col.remove(["test2", "test3"])
self.assertEqual(col._collection, [])
def test_filecoll_extend(self, mock_api):
"""Test extend"""
col = FileCollection(mock_api)
col2 = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
test_file2 = mock.create_autospec(UserFile)
col._collection = [test_file]
col2._collection = [test_file2, test_file]
with self.assertRaises(AttributeError):
col.extend(None)
with self.assertRaises(AttributeError):
col.extend("test")
with self.assertRaises(AttributeError):
col.extend([])
col.extend(col2)
self.assertEqual(len(col._collection), 2)
self.assertTrue(all(i in [test_file, test_file2]
for i in col._collection))
col2.extend(col)
self.assertEqual(len(col._collection), 2)
self.assertTrue(all(i in [test_file, test_file2]
for i in col._collection))
def test_filecoll_add(self, mock_api):
"""Test add"""
col = FileCollection(mock_api)
test_file = mock.create_autospec(UserFile)
with self.assertRaises(FileInvalidException):
col.add("test")
with self.assertRaises(FileInvalidException):
col.add(1)
with self.assertRaises(FileInvalidException):
col.add(None)
col.add(test_file)
self.assertEqual(col._collection, [test_file])
with self.assertRaises(FileInvalidException):
col.add(test_file)
col._collection = []
col.add([test_file])
self.assertEqual(col._collection, [test_file])
col.add([test_file])
self.assertEqual(col._collection, [test_file])
col._collection = []
col.add([test_file, test_file])
col._collection = []
col.add([1, "2", None])
self.assertEqual(col._collection, [])
class JobSubmission(object):
"""
Description of a new processing request to be sent to the cloud.
Specifies application to be used, the files required for processing and
any additional parameters required for the processing.
It also specifies the requested compute resources to be dedicated to the
job on submission.
:Attributes:
- name (str)
- required_files (:py:class:`.FileCollection`)
- source (str)
- instances (int)
- pool (:py:class:`.Pool`)
- params (dict)
- settings (str)
"""
def __init__(self, client, job_name, **job_settings):
"""
:Args:
- client (:py:class:`.BatchAppsApi`): A configured and
authenticated API instance.
- job_name (str): A name for the new job.
:Kwargs:
- job_settings (dict): *Optional* Additional job settings or
parameters can be added as keyword arguments. These include:
- 'params': A string dict of job parameters to add to the
submission.
- 'files': A :py:class:`.FileCollection` of required files
to include with the job.
- 'job_file': The name (str) of the file that should be
used to start the job. This filename should be
included in the above ``files`` collection.
- 'instances': The number (int) of instances to allocate
to the job on submission.
- 'pool': A :py:class:`.Pool` to submit the job to. Default
is ``None``, i.e. create an auto-pool.
"""
if not hasattr(client, 'send_job'):
raise TypeError(
"client must be an authenticated BatchAppsApi object.")
super(JobSubmission, self).__setattr__(
'_api', client)
super(JobSubmission, self).__setattr__(
'_log', logging.getLogger('batch_apps'))
super(JobSubmission, self).__setattr__(
'name', str(job_name))
super(JobSubmission, self).__setattr__(
'params', job_settings.get('params', self.get_default_params()))
super(JobSubmission, self).__setattr__(
'required_files', job_settings.get('files', None))
super(JobSubmission, self).__setattr__(
'source', str(job_settings.get('job_file', ""))) #DEP
super(JobSubmission, self).__setattr__(
'instances', int(job_settings.get('instances', 0))) #DEP
super(JobSubmission, self).__setattr__(
'pool', job_settings.get('pool', None))
super(JobSubmission, self).__setattr__(
'settings', str(job_settings.get('settings', "")))
def __str__(self):
"""Job submission as a string
Returns:
- A string of the jobsubmission object as dict
"""
return str(self.__dict__)
def __getattr__(self, name):
"""
Get a job attribute, or a job parameter.
This will only be called if :py:class:`.JobSubmission` object does
not have the requested attribute, in which case the job parameters
dictionary will be searched for a matching key.
:Returns:
- The value (str) of the parameter if it is found.
:Raises:
- :py:exc:`AttributeError` if the requested attribute/parameter
does not exist.
"""
#TODO: Make all parameters case-insensitive.
try:
return super(
JobSubmission,
self).__getattribute__('params')[str(name)]
except KeyError:
raise AttributeError("'JobSubmission' object has no attribute or "
"parameter: {atr}".format(atr=name))
def __setattr__(self, name, value):
"""
Set a job attribute if it exists, or add a job parameter.
If the :py:class:`.JobSubmission` object has the named attribute
this will be set. If no such attribute exists, the key and value will
be added as a string pair to the job parameters dictionary.
:Args:
- name (str): The name of the attribute/parameter to be set.
- value: The value of the attribute/parameter to set to. If this
value is added as a parameter, it will be converted to a string
regardless of its initial type.
"""
if hasattr(self, name):
super(JobSubmission, self).__setattr__(name, value)
else:
self.params[str(name)] = str(value) #TODO: resolve parameter cases
def __delattr__(self, name):
"""Clear job attribute or delete parameter if it exists
:Args:
- name (str): The name of the attribute/parameter to wipe.
:Raises:
- :py:class:`AttributeError` if the :py:class:`.JobSubmission`
object has no attribute or parameter of that name.
"""
try:
super(JobSubmission, self).__delattr__(name)
return
except AttributeError:
pass
try:
del self.params[str(name)] # TODO: resolve parameter cases
except KeyError:
raise AttributeError("'JobSubmission' object has no attribute or "
"parameter: {atr}".format(atr=name))
def _filter_params(self):
"""
Parses job parameters before submission.
Checks the job submission parameters against the defaults for that
application, and adds additional parameters where necessary.
The new dictionary is formatted for the REST client (See
:py:func:`batchapps.utils.format_dictionary()`).
:Returns:
- Updated, formatted, parameters dictionary after cross-referencing
against defaults.
"""
default_params = self.get_default_params()
complete_params = dict(self.get_default_params())
complete_params.update(self.params)
return utils.format_dictionary(complete_params)
def _auto_pool(self, size):
"""
Create an autopoolspecification reference for use in job
submission.
:Returns:
- A dictionary.
"""
pool = PoolSpecifier(self._api, target_size=size)
return {
'targetDedicated': str(pool.target_size),
'maxTasksPerTVM': str(pool.max_tasks),
'communication': pool.communication,
'certificateReferences': pool.certificates
}
def _create_job_message(self):
"""
Create a job message for submitting to the REST API.
Only used internally on job submission (see :py:meth:.submit()).
:Returns:
- Dictionary of the job submission formatted for the REST API.
"""
#TODO: Final check of source file, add xml settings, allow for user
# to set priority, verify all job data is correct format
if not hasattr(self.required_files, '_get_message'):
self.add_file_collection()
if self.pool and hasattr(self.pool, 'id'):
pool_options = {'poolId': self.pool.id}
elif self.pool:
pool_options = {'poolId': str(self.pool)}
else:
size = max(int(self.instances), 1)
pool_options = {'autoPoolSpecification': self._auto_pool(size)}
job_message = {
'Name': str(self.name),
'Type': self._api.jobtype(),
'RequiredFiles': self.required_files._get_message("submit"),
'Parameters': list(self._filter_params()),
'JobFile': str(self.source),
'Settings': str(self.settings),
'Priority': 'Medium'
}
job_message.update(pool_options)
self._log.debug("Job message: {0}".format(job_message))
return job_message
def add_file_collection(self, file_collection=None):
"""
Assign a file collection to the job.
The userfiles assigned to a job submission will be loaded onto each
node prior to the job being executed.
:Kwargs:
- file_collection (:py:class:`.FileCollection`): If set, this will
be assigned as the :py:class:`.FileCollection` of the job. If
not set, a new :py:class:`.FileCollection` will be created.
:Raises:
- :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
passed in.
"""
if file_collection is None:
self._log.info("Assigning empty FileCollection to job")
self.required_files = FileCollection(self._api)
elif hasattr(file_collection, 'add'):
self._log.debug("Assigning FileCollection with {0} "
"userfiles to job".format(len(file_collection)))
self.required_files = file_collection
else:
raise TypeError("Can only assign an object of type FileCollection"
", not {type}".format(type=type(file_collection)))
def get_default_params(self):
"""
Get default parameters.
Get the parameters specified in the :class:`.Configuration` for the
current application.
:Returns:
- The parameters as a dictionary of strings.
"""
return self._api.default_params()
def add_file(self, userfile):
"""
Add userfile to required files list.
If the job does not have an :class:`.FileCollection` already assigned,
a new one will be created.
:Args:
- userfile (:class:`.UserFile`): The userfile to be added to
the job.
"""
if self.required_files is None:
self.required_files = FileCollection(self._api)
self.required_files.add(userfile)
def set_job_file(self, jobfile):
"""
Set file as the source from which the job will be started.
This will be the file that is executed to started the job.
:Args:
- jobfile (:class:`.UserFile`, int): The userfile to be used. This
can also be the index of a userfile in the collection, or it
can be an :class:`.UserFile` object.
If a new :class:`.UserFile` object is passed in, it will also
be added to the required files collection.
:Raises:
- :exc:`ValueError` if ``jobfile`` is not an in-range index,
or of an invalid type.
"""
if self.required_files is None:
raise ValueError("This job has no associated FileCollection.")
if hasattr(jobfile, "create_query_specifier"):
if jobfile not in self.required_files:
self._log.info("Assigned job file not in collection, "
"adding to required files")
self.required_files.add(jobfile)
self.source = jobfile.name
elif isinstance(jobfile, int) and jobfile < len(self.required_files):
self.source = self.required_files[jobfile].name
else:
raise ValueError(
"No job file to match {0} could be found.".format(jobfile))
self._log.debug(
"Assigned file: {0} as starting job file".format(self.source))
def submit(self):
"""Submit the job.
:Returns:
- If successful, a dictionary holding the new job's ID and a URL
to get the job details (See: :meth:`.SubmittedJob.update()`).
Dictionary has the keys: ``['jobId', 'link']``
.. warning:: 'jobId' key will be deprecated to be replaced with 'id'.
:Raises:
- :class:`.RestCallException` if job submission failed.
"""
resp = self._api.send_job(self._create_job_message())
if resp.success:
self._log.info("Job successfully submitted with ID: "
"{0}".format(resp.result['jobId']))
return {'jobId':resp.result['jobId'], #DEP
'id': resp.result['jobId'],
'link': resp.result['link']['href']}
else:
raise resp.result
class JobSubmission(object):
"""
Description of a new processing request to be sent to the cloud.
Specifies application to be used, the files required for processing and
any additional parameters required for the processing.
It also specifies the requested compute resources to be dedicated to the
job on submission.
:Attributes:
- name (str)
- required_files (:py:class:`.FileCollection`)
- source (str)
- instances (int)
- pool (:py:class:`.Pool`)
- params (dict)
- settings (str)
"""
def __init__(self, client, job_name, **job_settings):
"""
:Args:
- client (:py:class:`.BatchAppsApi`): A configured and
authenticated API instance.
- job_name (str): A name for the new job.
:Kwargs:
- job_settings (dict): *Optional* Additional job settings or
parameters can be added as keyword arguments. These include:
- 'params': A string dict of job parameters to add to the
submission.
- 'files': A :py:class:`.FileCollection` of required files
to include with the job.
- 'job_file': The name (str) of the file that should be
used to start the job. This filename should be
included in the above ``files`` collection.
- 'instances': The number (int) of instances to allocate
to the job on submission.
- 'pool': A :py:class:`.Pool` to submit the job to. Default
is ``None``, i.e. create an auto-pool.
"""
if not hasattr(client, 'send_job'):
raise TypeError(
"client must be an authenticated BatchAppsApi object.")
super(JobSubmission, self).__setattr__('_api', client)
super(JobSubmission, self).__setattr__('_log',
logging.getLogger('batch_apps'))
super(JobSubmission, self).__setattr__('name', str(job_name))
super(JobSubmission, self).__setattr__(
'params', job_settings.get('params', self.get_default_params()))
super(JobSubmission, self).__setattr__('required_files',
job_settings.get('files', None))
super(JobSubmission,
self).__setattr__('source', str(job_settings.get('job_file',
""))) #DEP
super(JobSubmission,
self).__setattr__('instances',
int(job_settings.get('instances', 0))) #DEP
super(JobSubmission, self).__setattr__('pool',
job_settings.get('pool', None))
super(JobSubmission,
self).__setattr__('settings',
str(job_settings.get('settings', "")))
def __str__(self):
"""Job submission as a string
Returns:
- A string of the jobsubmission object as dict
"""
return str(self.__dict__)
def __getattr__(self, name):
"""
Get a job attribute, or a job parameter.
This will only be called if :py:class:`.JobSubmission` object does
not have the requested attribute, in which case the job parameters
dictionary will be searched for a matching key.
:Returns:
- The value (str) of the parameter if it is found.
:Raises:
- :py:exc:`AttributeError` if the requested attribute/parameter
does not exist.
"""
#TODO: Make all parameters case-insensitive.
try:
return super(JobSubmission,
self).__getattribute__('params')[str(name)]
except KeyError:
raise AttributeError("'JobSubmission' object has no attribute or "
"parameter: {atr}".format(atr=name))
def __setattr__(self, name, value):
"""
Set a job attribute if it exists, or add a job parameter.
If the :py:class:`.JobSubmission` object has the named attribute
this will be set. If no such attribute exists, the key and value will
be added as a string pair to the job parameters dictionary.
:Args:
- name (str): The name of the attribute/parameter to be set.
- value: The value of the attribute/parameter to set to. If this
value is added as a parameter, it will be converted to a string
regardless of its initial type.
"""
if hasattr(self, name):
super(JobSubmission, self).__setattr__(name, value)
else:
self.params[str(name)] = str(value) #TODO: resolve parameter cases
def __delattr__(self, name):
"""Clear job attribute or delete parameter if it exists
:Args:
- name (str): The name of the attribute/parameter to wipe.
:Raises:
- :py:class:`AttributeError` if the :py:class:`.JobSubmission`
object has no attribute or parameter of that name.
"""
try:
super(JobSubmission, self).__delattr__(name)
return
except AttributeError:
pass
try:
del self.params[str(name)] # TODO: resolve parameter cases
except KeyError:
raise AttributeError("'JobSubmission' object has no attribute or "
"parameter: {atr}".format(atr=name))
def _filter_params(self):
"""
Parses job parameters before submission.
Checks the job submission parameters against the defaults for that
application, and adds additional parameters where necessary.
The new dictionary is formatted for the REST client (See
:py:func:`batchapps.utils.format_dictionary()`).
:Returns:
- Updated, formatted, parameters dictionary after cross-referencing
against defaults.
"""
default_params = self.get_default_params()
complete_params = dict(self.get_default_params())
complete_params.update(self.params)
return utils.format_dictionary(complete_params)
def _auto_pool(self, size):
"""
Create an autopoolspecification reference for use in job
submission.
:Returns:
- A dictionary.
"""
pool = PoolSpecifier(self._api, target_size=size)
return {
'targetDedicated': str(pool.target_size),
'maxTasksPerTVM': str(pool.max_tasks),
'communication': pool.communication,
'certificateReferences': pool.certificates
}
def _create_job_message(self):
"""
Create a job message for submitting to the REST API.
Only used internally on job submission (see :py:meth:.submit()).
:Returns:
- Dictionary of the job submission formatted for the REST API.
"""
#TODO: Final check of source file, add xml settings, allow for user
# to set priority, verify all job data is correct format
if not hasattr(self.required_files, '_get_message'):
self.add_file_collection()
if self.pool and hasattr(self.pool, 'id'):
pool_options = {'poolId': self.pool.id}
elif self.pool:
pool_options = {'poolId': str(self.pool)}
else:
size = max(int(self.instances), 1)
pool_options = {'autoPoolSpecification': self._auto_pool(size)}
job_message = {
'Name': str(self.name),
'Type': self._api.jobtype(),
'RequiredFiles': self.required_files._get_message("submit"),
'Parameters': list(self._filter_params()),
'JobFile': str(self.source),
'Settings': str(self.settings),
'Priority': 'Medium'
}
job_message.update(pool_options)
self._log.debug("Job message: {0}".format(job_message))
return job_message
def add_file_collection(self, file_collection=None):
"""
Assign a file collection to the job.
The userfiles assigned to a job submission will be loaded onto each
node prior to the job being executed.
:Kwargs:
- file_collection (:py:class:`.FileCollection`): If set, this will
be assigned as the :py:class:`.FileCollection` of the job. If
not set, a new :py:class:`.FileCollection` will be created.
:Raises:
- :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
passed in.
"""
if file_collection is None:
self._log.info("Assigning empty FileCollection to job")
self.required_files = FileCollection(self._api)
elif hasattr(file_collection, 'add'):
self._log.debug("Assigning FileCollection with {0} "
"userfiles to job".format(len(file_collection)))
self.required_files = file_collection
else:
raise TypeError("Can only assign an object of type FileCollection"
", not {type}".format(type=type(file_collection)))
def get_default_params(self):
"""
Get default parameters.
Get the parameters specified in the :class:`.Configuration` for the
current application.
:Returns:
- The parameters as a dictionary of strings.
"""
return self._api.default_params()
def add_file(self, userfile):
"""
Add userfile to required files list.
If the job does not have an :class:`.FileCollection` already assigned,
a new one will be created.
:Args:
- userfile (:class:`.UserFile`): The userfile to be added to
the job.
"""
if self.required_files is None:
self.required_files = FileCollection(self._api)
self.required_files.add(userfile)
def set_job_file(self, jobfile):
"""
Set file as the source from which the job will be started.
This will be the file that is executed to started the job.
:Args:
- jobfile (:class:`.UserFile`, int): The userfile to be used. This
can also be the index of a userfile in the collection, or it
can be an :class:`.UserFile` object.
If a new :class:`.UserFile` object is passed in, it will also
be added to the required files collection.
:Raises:
- :exc:`ValueError` if ``jobfile`` is not an in-range index,
or of an invalid type.
"""
if self.required_files is None:
raise ValueError("This job has no associated FileCollection.")
if hasattr(jobfile, "create_query_specifier"):
if jobfile not in self.required_files:
self._log.info("Assigned job file not in collection, "
"adding to required files")
self.required_files.add(jobfile)
self.source = jobfile.name
elif isinstance(jobfile, int) and jobfile < len(self.required_files):
self.source = self.required_files[jobfile].name
else:
raise ValueError(
"No job file to match {0} could be found.".format(jobfile))
self._log.debug("Assigned file: {0} as starting job file".format(
self.source))
def submit(self):
"""Submit the job.
:Returns:
- If successful, a dictionary holding the new job's ID and a URL
to get the job details (See: :meth:`.SubmittedJob.update()`).
Dictionary has the keys: ``['jobId', 'link']``
.. warning:: 'jobId' key will be deprecated to be replaced with 'id'.
:Raises:
- :class:`.RestCallException` if job submission failed.
"""
resp = self._api.send_job(self._create_job_message())
if resp.success:
self._log.info("Job successfully submitted with ID: "
"{0}".format(resp.result['jobId']))
return {
'jobId': resp.result['jobId'], #DEP
'id': resp.result['jobId'],
'link': resp.result['link']['href']
}
else:
raise resp.result