示例#1
0
    def add_file_collection(self, file_collection=None):
        """
        Assign a file collection to the job.
        The userfiles assigned to a job submission will be loaded onto each
        node prior to the job being executed.

        :Kwargs:
            - file_collection (:py:class:`.FileCollection`): If set, this will
              be assigned as the :py:class:`.FileCollection` of the job. If
              not set, a new :py:class:`.FileCollection` will be created.

        :Raises:
            - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
              passed in.
        """
        if file_collection is None:
            self._log.info("Assigning empty FileCollection to job")
            self.required_files = FileCollection(self._api)

        elif hasattr(file_collection, 'add'):
            self._log.debug("Assigning FileCollection with {0} "
                            "userfiles to job".format(len(file_collection)))

            self.required_files = file_collection

        else:
            raise TypeError("Can only assign an object of type FileCollection"
                            ", not {type}".format(type=type(file_collection)))
    def test_filecoll_str(self, mock_api, mock_add):
        """Test __str__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        colstr = str(col)
        self.assertEqual(colstr, "['1', 'None', 'test', '[]', '{}']")
    def test_filecoll_len(self, mock_api, mock_add):
        """Test __len__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        self.assertEqual(len(col), len(col._collection))

        col._collection.append("more")
        self.assertEqual(len(col), len(col._collection))
示例#4
0
    def add_file(self, userfile):
        """
        Add userfile to required files list.
        If the job does not have an :class:`.FileCollection` already assigned,
        a new one will be created.

        :Args:
            - userfile (:class:`.UserFile`): The userfile to be added to
              the job.
        """
        if self.required_files is None:
            self.required_files = FileCollection(self._api)

        self.required_files.add(userfile)
示例#5
0
    def add_file_collection(self, file_collection=None):
        """
        Assign a file collection to the job.
        The userfiles assigned to a job submission will be loaded onto each
        node prior to the job being executed.

        :Kwargs:
            - file_collection (:py:class:`.FileCollection`): If set, this will
              be assigned as the :py:class:`.FileCollection` of the job. If
              not set, a new :py:class:`.FileCollection` will be created.

        :Raises:
            - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
              passed in.
        """
        if file_collection is None:
            self._log.info("Assigning empty FileCollection to job")
            self.required_files = FileCollection(self._api)

        elif hasattr(file_collection, 'add'):
            self._log.debug("Assigning FileCollection with {0} "
                            "userfiles to job".format(len(file_collection)))

            self.required_files = file_collection

        else:
            raise TypeError("Can only assign an object of type FileCollection"
                            ", not {type}".format(type=type(file_collection)))
    def test_filecoll_iter(self, mock_api, mock_add):
        """Test __iter__"""

        col = FileCollection(mock_api)

        itr = iter(col)
        with self.assertRaises(StopIteration):
            next(itr)

        col._collection.append(None)
        for ufile in col:
            self.assertIsNone(ufile)
    def test_filecoll_delitem(self, mock_api, mock_add):
        """Test __delitem__"""

        col = FileCollection(mock_api)
        col._collection = [1, None, "test", [], {}]

        del col[0]
        del col[-1]
        del col[1:]
        self.assertEqual(col._collection, [None])

        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"
        col._collection = [test_file]

        del col["test"]
        self.assertEqual(col._collection, [])
        del col["something"]
        self.assertEqual(col._collection, [])

        del col[5]
        del col[None]
        del col[0:-1]
示例#8
0
    def add_file(self, userfile):
        """
        Add userfile to required files list.
        If the job does not have an :class:`.FileCollection` already assigned,
        a new one will be created.

        :Args:
            - userfile (:class:`.UserFile`): The userfile to be added to
              the job.
        """
        if self.required_files is None:
            self.required_files = FileCollection(self._api)

        self.required_files.add(userfile)
    def test_filecoll_index(self, mock_api):
        """Test index"""

        col = FileCollection(mock_api)

        test_file = mock.create_autospec(UserFile)
        test_file2 = mock.create_autospec(UserFile)
        test_file3 = mock.create_autospec(UserFile)

        col._collection = [test_file, test_file2]

        with self.assertRaises(TypeError):
            col.index(None)
        with self.assertRaises(TypeError):
            col.index("test")

        with self.assertRaises(ValueError):
            col.index(test_file3)

        self.assertEqual(col._collection.index(test_file2), 1)
    def test_filecoll_get_message(self, mock_api, mock_add):
        """Test _get_message"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)
        test_file.create_query_specifier.return_value = {"test_query":1}
        test_file.create_submit_specifier.return_value = {"test_submit":2}
        col._collection = [test_file]

        specs = col._get_message(None)
        self.assertEqual(specs, [])

        specs = col._get_message(1)
        self.assertEqual(specs, [])

        specs = col._get_message([])
        self.assertEqual(specs, [])

        specs = col._get_message("query")
        self.assertEqual(specs, [{"test_query":1}])
        specs = col._get_message("submit")
        self.assertEqual(specs, [{"test_submit":2}])
    def test_filecoll_upload(self, mock_isup, mock_upload, mock_api):
        """Test upload"""

        _callback = mock.Mock()
        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)
        mock_isup.return_value = []
        mock_upload.return_value = (False, "f", "Error!")

        mock_isup.called = False
        col = FileCollection(mock_api)
        failed = col.upload()
        self.assertTrue(mock_isup.called)
        self.assertFalse(mock_upload.called)
        self.assertEqual(failed, [])

        mock_isup.called = False
        failed = col.upload(force=True)
        self.assertFalse(mock_isup.called)
        self.assertFalse(mock_upload.called)
        self.assertEqual(failed, [])

        col._collection = [1, 2, 3, 4]
        failed = col.upload(force=True)
        mock_upload.assert_any_call(1, callback=None, block=4096)
        self.assertEqual(mock_upload.call_count, 4)

        self.assertEqual(failed, [("f", "Error!"),
                                  ("f", "Error!"),
                                  ("f", "Error!"),
                                  ("f", "Error!")])

        mock_upload.call_count = 0
        resp.success = True
        mock_upload.return_value = (True, "f", "All good!")
        failed = col.upload(force=True, threads=None, callback=_callback, block=1)
        mock_upload.assert_any_call(1, callback=_callback, block=1)
        self.assertEqual(mock_upload.call_count, 4)
        self.assertEqual(failed, [])
    def test_filecoll_getitem(self, mock_api, mock_add):
        """Test __getitem__"""

        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"

        col = FileCollection(mock_api)
        with self.assertRaises(FileMissingException):
            print(col[1])

        self.assertEqual(col[:1], [])
        col._collection.append(test_file)

        self.assertEqual(col[0], test_file)
        self.assertEqual(col["test"], [test_file])
        self.assertEqual(col[-1], test_file)
        self.assertEqual(col[:1], [test_file])

        with self.assertRaises(FileMissingException):
            print(col[10])
        with self.assertRaises(FileMissingException):
            print(col["test2"])
        with self.assertRaises(FileMissingException):
            print(col[None])
    def test_filecoll_create(self, mock_api, mock_add):
        """Test FileCollection object"""

        with self.assertRaises(TypeError):
            FileCollection(None)

        FileCollection(mock_api)
        self.assertFalse(mock_add.called)

        FileCollection(mock_api, "test")
        mock_add.assert_called_once_with("test")

        FileCollection(mock_api, "test1", "test2")
        mock_add.assert_any_call("test1")
        mock_add.assert_any_call("test2")

        FileCollection(mock_api, ["test1", "test2"])
        mock_add.assert_any_call("test1")
        mock_add.assert_any_call("test2")

        FileCollection(mock_api, None)
        mock_add.assert_any_call(None)
    def test_filecoll_upload_thread(self, mock_pik, mock_api):
        """Test upload"""

        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)

        col = FileCollection(mock_api)
        col._api = None
        failed = col.upload(force=True, threads=1)
        self.assertFalse(mock_pik.called)
        self.assertEqual(failed, [])

        col._collection = [1, 2, 3, 4]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, 
                         [(1, "'int' object has no attribute 'upload'"),
                          (2, "'int' object has no attribute 'upload'"),
                          (3, "'int' object has no attribute 'upload'"),
                          (4, "'int' object has no attribute 'upload'")])

        mock_pik.call_count = 0
        col._collection = [UFile()]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(len(failed), 1)
        self.assertIsInstance(failed[0], tuple)

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True)]
        failed = col.upload(force=True, threads=1)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, [])

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True)]
        failed = col.upload(force=True, threads=3)
        self.assertEqual(mock_pik.call_count, 1)
        self.assertEqual(failed, [])

        mock_pik.call_count = 0
        col._collection = [UFile() for a in range(15)]
        failed = col.upload(force=True, threads=3)
        self.assertEqual(mock_pik.call_count, 5)
        self.assertEqual(len(failed), 15)

        mock_pik.call_count = 0
        col._collection = [UFile(arg_a=True) for a in range(20)]
        failed = col.upload(force=True, threads=20)
        self.assertEqual(mock_pik.call_count, 2)
        self.assertEqual(failed, [])
    def test_filecoll_is_uploaded(self,
                                  mock_rem,
                                  mock_mess,
                                  mock_ufile,
                                  mock_api):
        """Test is_uploaded"""

        def user_file_gen(u_name):
            """Mock UserFile generator"""
            ugen = mock.create_autospec(UserFile)
            ugen.name = str(u_name)
            ugen.compare_lastmodified.return_value = True
            return ugen

        def add(col, itm):
            """Mock add UserFile to collection"""
            col._collection.append(itm)

        resp = mock.create_autospec(Response)
        resp.success = False
        resp.result = RestCallException(None, "Boom", None)
        mock_ufile.return_value = user_file_gen("1")
        FileCollection.add = add
        mock_api.query_files.return_value = resp
        mock_mess.return_value = ["1", "2", "3", "4", "5"]

        col = FileCollection(mock_api)
        upl = col.is_uploaded()
        self.assertIsInstance(upl, FileCollection)
        self.assertEqual(upl._collection, col._collection)
        self.assertFalse(mock_api.query_files.called)

        col._collection = [1, 2, 3, 4, 5]
        with self.assertRaises(RestCallException):
            col.is_uploaded()
        mock_api.query_files.assert_called_once_with(["1", "2", "3", "4", "5"])

        with self.assertRaises(RestCallException):
            col.is_uploaded(per_call=2)
        mock_api.query_files.assert_called_with(["1", "2"])

        col._collection = [user_file_gen("1"), user_file_gen("2")]
        mock_api.reset()
        resp.success = True
        resp.result = ["test1", "test2", "test3"]
        upl = col.is_uploaded()
        mock_api.query_files.assert_called_with(["1", "2", "3", "4", "5"])
        mock_rem.assert_called_with([mock.ANY])
        self.assertEqual(upl._collection, col._collection)

        col._collection = [user_file_gen("test1"), user_file_gen("test2")]
        upl = col.is_uploaded()
        mock_rem.assert_called_with([])
        self.assertEqual(upl._collection, col._collection)
    def test_filecoll_remove(self, mock_api):
        """Test remove"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)
        test_file.name = "test"
        col._collection = [test_file, 1, "2", None, []]

        with self.assertRaises(TypeError):
            col.remove(None)
        with self.assertRaises(TypeError):
            col.remove(10)

        col.remove(1)
        col.remove(-1)
        col.remove(slice(1))
        self.assertEqual(col._collection, ["2", None])

        test_file2 = mock.create_autospec(UserFile)
        test_file2.name = "test2"
        test_file3 = mock.create_autospec(UserFile)
        test_file3.name = "test3"
        col._collection = [test_file, test_file2, test_file3]
        col.remove("test")
        self.assertEqual(col._collection, [test_file2, test_file3])
        col.remove(["test2", "test3"])
        self.assertEqual(col._collection, [])
    def test_filecoll_extend(self, mock_api):
        """Test extend"""

        col = FileCollection(mock_api)
        col2 = FileCollection(mock_api)

        test_file = mock.create_autospec(UserFile)
        test_file2 = mock.create_autospec(UserFile)

        col._collection = [test_file]
        col2._collection = [test_file2, test_file]

        with self.assertRaises(AttributeError):
            col.extend(None)
        with self.assertRaises(AttributeError):
            col.extend("test")
        with self.assertRaises(AttributeError):
            col.extend([])

        col.extend(col2)
        self.assertEqual(len(col._collection), 2)
        self.assertTrue(all(i in [test_file, test_file2]
                            for i in col._collection))

        col2.extend(col)
        self.assertEqual(len(col._collection), 2)
        self.assertTrue(all(i in [test_file, test_file2]
                            for i in col._collection))
    def test_filecoll_add(self, mock_api):
        """Test add"""

        col = FileCollection(mock_api)
        test_file = mock.create_autospec(UserFile)

        with self.assertRaises(FileInvalidException):
            col.add("test")
        with self.assertRaises(FileInvalidException):
            col.add(1)
        with self.assertRaises(FileInvalidException):
            col.add(None)

        col.add(test_file)
        self.assertEqual(col._collection, [test_file])
        with self.assertRaises(FileInvalidException):
            col.add(test_file)

        col._collection = []
        col.add([test_file])
        self.assertEqual(col._collection, [test_file])
        col.add([test_file])
        self.assertEqual(col._collection, [test_file])

        col._collection = []
        col.add([test_file, test_file])

        col._collection = []
        col.add([1, "2", None])
        self.assertEqual(col._collection, [])
示例#19
0
class JobSubmission(object):
    """
    Description of a new processing request to be sent to the cloud.
    Specifies application to be used, the files required for processing and
    any additional parameters required for the processing.
    It also specifies the requested compute resources to be dedicated to the
    job on submission.

    :Attributes:
        - name (str)
        - required_files (:py:class:`.FileCollection`)
        - source (str)
        - instances (int)
        - pool (:py:class:`.Pool`)
        - params (dict)
        - settings (str)
    """

    def __init__(self, client, job_name, **job_settings):
        """
        :Args:
            - client (:py:class:`.BatchAppsApi`): A configured and
              authenticated API instance.
            - job_name (str): A name for the new job.

        :Kwargs:
            - job_settings (dict): *Optional* Additional job settings or
              parameters can be added as keyword arguments. These include:

                  - 'params': A string dict of job parameters to add to the
                    submission.
                  - 'files': A :py:class:`.FileCollection` of required files
                    to include with the job.
                  - 'job_file': The name (str) of the file that should be
                    used to start the job. This filename should be
                    included in the above ``files`` collection.
                  - 'instances': The number (int) of instances to allocate
                    to the job on submission.
                  - 'pool': A :py:class:`.Pool` to submit the job to. Default
                    is ``None``, i.e. create an auto-pool.
        """
        if not hasattr(client, 'send_job'):
            raise TypeError(
                "client must be an authenticated BatchAppsApi object.")

        super(JobSubmission, self).__setattr__(
            '_api', client)
        super(JobSubmission, self).__setattr__(
            '_log', logging.getLogger('batch_apps'))

        super(JobSubmission, self).__setattr__(
            'name', str(job_name))
        super(JobSubmission, self).__setattr__(
            'params', job_settings.get('params', self.get_default_params()))
        super(JobSubmission, self).__setattr__(
            'required_files', job_settings.get('files', None))
        super(JobSubmission, self).__setattr__(
            'source', str(job_settings.get('job_file', ""))) #DEP
        super(JobSubmission, self).__setattr__(
            'instances', int(job_settings.get('instances', 0))) #DEP
        super(JobSubmission, self).__setattr__(
            'pool', job_settings.get('pool', None))
        super(JobSubmission, self).__setattr__(
            'settings', str(job_settings.get('settings', "")))

    def __str__(self):
        """Job submission as a string

        Returns:
            - A string of the jobsubmission object as dict
        """
        return str(self.__dict__)

    def __getattr__(self, name):
        """
        Get a job attribute, or a job parameter.
        This will only be called if :py:class:`.JobSubmission` object does
        not have the requested attribute, in which case the job parameters
        dictionary will be searched for a matching key.

        :Returns:
            - The value (str) of the parameter if it is found.

        :Raises:
            - :py:exc:`AttributeError` if the requested attribute/parameter
              does not exist.
        """
        #TODO: Make all parameters case-insensitive.
        try:
            return super(
                JobSubmission,
                self).__getattribute__('params')[str(name)]

        except KeyError:
            raise AttributeError("'JobSubmission' object has no attribute or "
                                 "parameter: {atr}".format(atr=name))

    def __setattr__(self, name, value):
        """
        Set a job attribute if it exists, or add a job parameter.
        If the :py:class:`.JobSubmission` object has the named attribute
        this will be set. If no such attribute exists, the key and value will
        be added as a string pair to the job parameters dictionary.

        :Args:
            - name (str): The name of the attribute/parameter to be set.
            - value: The value of the attribute/parameter to set to. If this
              value is added as a parameter, it will be converted to a string
              regardless of its initial type.

        """
        if hasattr(self, name):
            super(JobSubmission, self).__setattr__(name, value)

        else:
            self.params[str(name)] = str(value) #TODO: resolve parameter cases

    def __delattr__(self, name):
        """Clear job attribute or delete parameter if it exists

        :Args:
            - name (str): The name of the attribute/parameter to wipe.

        :Raises:
            - :py:class:`AttributeError` if the :py:class:`.JobSubmission`
              object has no attribute or parameter of that name.
        """
        try:
            super(JobSubmission, self).__delattr__(name)
            return

        except AttributeError:
            pass

        try:
            del self.params[str(name)] # TODO: resolve parameter cases

        except KeyError:
            raise AttributeError("'JobSubmission' object has no attribute or "
                                 "parameter: {atr}".format(atr=name))

    def _filter_params(self):
        """
        Parses job parameters before submission.
        Checks the job submission parameters against the defaults for that
        application, and adds additional parameters where necessary.
        The new dictionary is formatted for the REST client (See
        :py:func:`batchapps.utils.format_dictionary()`).

        :Returns:
            - Updated, formatted, parameters dictionary after cross-referencing
              against defaults.
        """
        default_params = self.get_default_params()
        complete_params = dict(self.get_default_params())
        complete_params.update(self.params)

        return utils.format_dictionary(complete_params)

    def _auto_pool(self, size):
        """
        Create an autopoolspecification reference for use in job
        submission.

        :Returns:
            - A dictionary.
        """
        pool = PoolSpecifier(self._api, target_size=size)

        return {
            'targetDedicated': str(pool.target_size),
            'maxTasksPerTVM': str(pool.max_tasks),
            'communication': pool.communication,
            'certificateReferences': pool.certificates
            }

    def _create_job_message(self):
        """
        Create a job message for submitting to the REST API.
        Only used internally on job submission (see :py:meth:.submit()).

        :Returns:
            - Dictionary of the job submission formatted for the REST API.
        """
        #TODO: Final check of source file, add xml settings, allow for user
        #      to set priority, verify all job data is correct format

        if not hasattr(self.required_files, '_get_message'):
            self.add_file_collection()

        if self.pool and hasattr(self.pool, 'id'):
            pool_options = {'poolId': self.pool.id}

        elif self.pool:
            pool_options = {'poolId': str(self.pool)}

        else:
            size = max(int(self.instances), 1)
            pool_options = {'autoPoolSpecification': self._auto_pool(size)}

        job_message = {
            'Name': str(self.name),
            'Type': self._api.jobtype(),
            'RequiredFiles': self.required_files._get_message("submit"),
            'Parameters': list(self._filter_params()),
            'JobFile': str(self.source),
            'Settings': str(self.settings),
            'Priority': 'Medium'
        }
        job_message.update(pool_options)

        self._log.debug("Job message: {0}".format(job_message))
        return job_message

    def add_file_collection(self, file_collection=None):
        """
        Assign a file collection to the job.
        The userfiles assigned to a job submission will be loaded onto each
        node prior to the job being executed.

        :Kwargs:
            - file_collection (:py:class:`.FileCollection`): If set, this will
              be assigned as the :py:class:`.FileCollection` of the job. If
              not set, a new :py:class:`.FileCollection` will be created.

        :Raises:
            - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
              passed in.
        """
        if file_collection is None:
            self._log.info("Assigning empty FileCollection to job")
            self.required_files = FileCollection(self._api)

        elif hasattr(file_collection, 'add'):
            self._log.debug("Assigning FileCollection with {0} "
                            "userfiles to job".format(len(file_collection)))

            self.required_files = file_collection

        else:
            raise TypeError("Can only assign an object of type FileCollection"
                            ", not {type}".format(type=type(file_collection)))

    def get_default_params(self):
        """
        Get default parameters.
        Get the parameters specified in the :class:`.Configuration` for the
        current application.

        :Returns:
            - The parameters as a dictionary of strings.
        """
        return self._api.default_params()

    def add_file(self, userfile):
        """
        Add userfile to required files list.
        If the job does not have an :class:`.FileCollection` already assigned,
        a new one will be created.

        :Args:
            - userfile (:class:`.UserFile`): The userfile to be added to
              the job.
        """
        if self.required_files is None:
            self.required_files = FileCollection(self._api)

        self.required_files.add(userfile)

    def set_job_file(self, jobfile):
        """
        Set file as the source from which the job will be started.
        This will be the file that is executed to started the job.

        :Args:
            - jobfile (:class:`.UserFile`, int): The userfile to be used. This
              can also be the index of a userfile in the collection, or it
              can be an :class:`.UserFile` object.
              If a new :class:`.UserFile` object is passed in, it will also
              be added to the required files collection.

        :Raises:
            - :exc:`ValueError` if ``jobfile`` is not an in-range index,
              or of an invalid type.
        """
        if self.required_files is None:
            raise ValueError("This job has no associated FileCollection.")

        if hasattr(jobfile, "create_query_specifier"):

            if jobfile not in self.required_files:
                self._log.info("Assigned job file not in collection, "
                               "adding to required files")

                self.required_files.add(jobfile)
            self.source = jobfile.name

        elif isinstance(jobfile, int) and jobfile < len(self.required_files):
            self.source = self.required_files[jobfile].name

        else:
            raise ValueError(
                "No job file to match {0} could be found.".format(jobfile))

        self._log.debug(
            "Assigned file: {0} as starting job file".format(self.source))

    def submit(self):
        """Submit the job.

        :Returns:
            - If successful, a dictionary holding the new job's ID and a URL
              to get the job details (See: :meth:`.SubmittedJob.update()`).
              Dictionary has the keys: ``['jobId', 'link']``

              .. warning:: 'jobId' key will be deprecated to be replaced with 'id'.

        :Raises:
            - :class:`.RestCallException` if job submission failed.
        """
        resp = self._api.send_job(self._create_job_message())
        
        if resp.success:
            self._log.info("Job successfully submitted with ID: "
                           "{0}".format(resp.result['jobId']))

            return {'jobId':resp.result['jobId'], #DEP
                    'id': resp.result['jobId'],
                    'link': resp.result['link']['href']}

        else:
            raise resp.result
示例#20
0
class JobSubmission(object):
    """
    Description of a new processing request to be sent to the cloud.
    Specifies application to be used, the files required for processing and
    any additional parameters required for the processing.
    It also specifies the requested compute resources to be dedicated to the
    job on submission.

    :Attributes:
        - name (str)
        - required_files (:py:class:`.FileCollection`)
        - source (str)
        - instances (int)
        - pool (:py:class:`.Pool`)
        - params (dict)
        - settings (str)
    """
    def __init__(self, client, job_name, **job_settings):
        """
        :Args:
            - client (:py:class:`.BatchAppsApi`): A configured and
              authenticated API instance.
            - job_name (str): A name for the new job.

        :Kwargs:
            - job_settings (dict): *Optional* Additional job settings or
              parameters can be added as keyword arguments. These include:

                  - 'params': A string dict of job parameters to add to the
                    submission.
                  - 'files': A :py:class:`.FileCollection` of required files
                    to include with the job.
                  - 'job_file': The name (str) of the file that should be
                    used to start the job. This filename should be
                    included in the above ``files`` collection.
                  - 'instances': The number (int) of instances to allocate
                    to the job on submission.
                  - 'pool': A :py:class:`.Pool` to submit the job to. Default
                    is ``None``, i.e. create an auto-pool.
        """
        if not hasattr(client, 'send_job'):
            raise TypeError(
                "client must be an authenticated BatchAppsApi object.")

        super(JobSubmission, self).__setattr__('_api', client)
        super(JobSubmission, self).__setattr__('_log',
                                               logging.getLogger('batch_apps'))

        super(JobSubmission, self).__setattr__('name', str(job_name))
        super(JobSubmission, self).__setattr__(
            'params', job_settings.get('params', self.get_default_params()))
        super(JobSubmission, self).__setattr__('required_files',
                                               job_settings.get('files', None))
        super(JobSubmission,
              self).__setattr__('source', str(job_settings.get('job_file',
                                                               "")))  #DEP
        super(JobSubmission,
              self).__setattr__('instances',
                                int(job_settings.get('instances', 0)))  #DEP
        super(JobSubmission, self).__setattr__('pool',
                                               job_settings.get('pool', None))
        super(JobSubmission,
              self).__setattr__('settings',
                                str(job_settings.get('settings', "")))

    def __str__(self):
        """Job submission as a string

        Returns:
            - A string of the jobsubmission object as dict
        """
        return str(self.__dict__)

    def __getattr__(self, name):
        """
        Get a job attribute, or a job parameter.
        This will only be called if :py:class:`.JobSubmission` object does
        not have the requested attribute, in which case the job parameters
        dictionary will be searched for a matching key.

        :Returns:
            - The value (str) of the parameter if it is found.

        :Raises:
            - :py:exc:`AttributeError` if the requested attribute/parameter
              does not exist.
        """
        #TODO: Make all parameters case-insensitive.
        try:
            return super(JobSubmission,
                         self).__getattribute__('params')[str(name)]

        except KeyError:
            raise AttributeError("'JobSubmission' object has no attribute or "
                                 "parameter: {atr}".format(atr=name))

    def __setattr__(self, name, value):
        """
        Set a job attribute if it exists, or add a job parameter.
        If the :py:class:`.JobSubmission` object has the named attribute
        this will be set. If no such attribute exists, the key and value will
        be added as a string pair to the job parameters dictionary.

        :Args:
            - name (str): The name of the attribute/parameter to be set.
            - value: The value of the attribute/parameter to set to. If this
              value is added as a parameter, it will be converted to a string
              regardless of its initial type.

        """
        if hasattr(self, name):
            super(JobSubmission, self).__setattr__(name, value)

        else:
            self.params[str(name)] = str(value)  #TODO: resolve parameter cases

    def __delattr__(self, name):
        """Clear job attribute or delete parameter if it exists

        :Args:
            - name (str): The name of the attribute/parameter to wipe.

        :Raises:
            - :py:class:`AttributeError` if the :py:class:`.JobSubmission`
              object has no attribute or parameter of that name.
        """
        try:
            super(JobSubmission, self).__delattr__(name)
            return

        except AttributeError:
            pass

        try:
            del self.params[str(name)]  # TODO: resolve parameter cases

        except KeyError:
            raise AttributeError("'JobSubmission' object has no attribute or "
                                 "parameter: {atr}".format(atr=name))

    def _filter_params(self):
        """
        Parses job parameters before submission.
        Checks the job submission parameters against the defaults for that
        application, and adds additional parameters where necessary.
        The new dictionary is formatted for the REST client (See
        :py:func:`batchapps.utils.format_dictionary()`).

        :Returns:
            - Updated, formatted, parameters dictionary after cross-referencing
              against defaults.
        """
        default_params = self.get_default_params()
        complete_params = dict(self.get_default_params())
        complete_params.update(self.params)

        return utils.format_dictionary(complete_params)

    def _auto_pool(self, size):
        """
        Create an autopoolspecification reference for use in job
        submission.

        :Returns:
            - A dictionary.
        """
        pool = PoolSpecifier(self._api, target_size=size)

        return {
            'targetDedicated': str(pool.target_size),
            'maxTasksPerTVM': str(pool.max_tasks),
            'communication': pool.communication,
            'certificateReferences': pool.certificates
        }

    def _create_job_message(self):
        """
        Create a job message for submitting to the REST API.
        Only used internally on job submission (see :py:meth:.submit()).

        :Returns:
            - Dictionary of the job submission formatted for the REST API.
        """
        #TODO: Final check of source file, add xml settings, allow for user
        #      to set priority, verify all job data is correct format

        if not hasattr(self.required_files, '_get_message'):
            self.add_file_collection()

        if self.pool and hasattr(self.pool, 'id'):
            pool_options = {'poolId': self.pool.id}

        elif self.pool:
            pool_options = {'poolId': str(self.pool)}

        else:
            size = max(int(self.instances), 1)
            pool_options = {'autoPoolSpecification': self._auto_pool(size)}

        job_message = {
            'Name': str(self.name),
            'Type': self._api.jobtype(),
            'RequiredFiles': self.required_files._get_message("submit"),
            'Parameters': list(self._filter_params()),
            'JobFile': str(self.source),
            'Settings': str(self.settings),
            'Priority': 'Medium'
        }
        job_message.update(pool_options)

        self._log.debug("Job message: {0}".format(job_message))
        return job_message

    def add_file_collection(self, file_collection=None):
        """
        Assign a file collection to the job.
        The userfiles assigned to a job submission will be loaded onto each
        node prior to the job being executed.

        :Kwargs:
            - file_collection (:py:class:`.FileCollection`): If set, this will
              be assigned as the :py:class:`.FileCollection` of the job. If
              not set, a new :py:class:`.FileCollection` will be created.

        :Raises:
            - :py:exc:`TypeError` if a non-:py:class:`.FileCollection` is
              passed in.
        """
        if file_collection is None:
            self._log.info("Assigning empty FileCollection to job")
            self.required_files = FileCollection(self._api)

        elif hasattr(file_collection, 'add'):
            self._log.debug("Assigning FileCollection with {0} "
                            "userfiles to job".format(len(file_collection)))

            self.required_files = file_collection

        else:
            raise TypeError("Can only assign an object of type FileCollection"
                            ", not {type}".format(type=type(file_collection)))

    def get_default_params(self):
        """
        Get default parameters.
        Get the parameters specified in the :class:`.Configuration` for the
        current application.

        :Returns:
            - The parameters as a dictionary of strings.
        """
        return self._api.default_params()

    def add_file(self, userfile):
        """
        Add userfile to required files list.
        If the job does not have an :class:`.FileCollection` already assigned,
        a new one will be created.

        :Args:
            - userfile (:class:`.UserFile`): The userfile to be added to
              the job.
        """
        if self.required_files is None:
            self.required_files = FileCollection(self._api)

        self.required_files.add(userfile)

    def set_job_file(self, jobfile):
        """
        Set file as the source from which the job will be started.
        This will be the file that is executed to started the job.

        :Args:
            - jobfile (:class:`.UserFile`, int): The userfile to be used. This
              can also be the index of a userfile in the collection, or it
              can be an :class:`.UserFile` object.
              If a new :class:`.UserFile` object is passed in, it will also
              be added to the required files collection.

        :Raises:
            - :exc:`ValueError` if ``jobfile`` is not an in-range index,
              or of an invalid type.
        """
        if self.required_files is None:
            raise ValueError("This job has no associated FileCollection.")

        if hasattr(jobfile, "create_query_specifier"):

            if jobfile not in self.required_files:
                self._log.info("Assigned job file not in collection, "
                               "adding to required files")

                self.required_files.add(jobfile)
            self.source = jobfile.name

        elif isinstance(jobfile, int) and jobfile < len(self.required_files):
            self.source = self.required_files[jobfile].name

        else:
            raise ValueError(
                "No job file to match {0} could be found.".format(jobfile))

        self._log.debug("Assigned file: {0} as starting job file".format(
            self.source))

    def submit(self):
        """Submit the job.

        :Returns:
            - If successful, a dictionary holding the new job's ID and a URL
              to get the job details (See: :meth:`.SubmittedJob.update()`).
              Dictionary has the keys: ``['jobId', 'link']``

              .. warning:: 'jobId' key will be deprecated to be replaced with 'id'.

        :Raises:
            - :class:`.RestCallException` if job submission failed.
        """
        resp = self._api.send_job(self._create_job_message())

        if resp.success:
            self._log.info("Job successfully submitted with ID: "
                           "{0}".format(resp.result['jobId']))

            return {
                'jobId': resp.result['jobId'],  #DEP
                'id': resp.result['jobId'],
                'link': resp.result['link']['href']
            }

        else:
            raise resp.result