def check_config_xml(self, contents): """ Check whether the given XML config file contents is well-formed and it has all the required parameters. :param string contents: the XML config file contents or XML config string :param bool is_config_string: if ``True``, contents is a config string :rtype: :class:`~aeneas.validator.ValidatorResult` """ self.log(u"Checking contents XML config file") self.result = ValidatorResult() if self._are_safety_checks_disabled(u"check_config_xml"): return self.result contents = gf.safe_bytes(contents) self.log(u"Checking that contents is well formed") self.check_raw_string(contents, is_bstring=True) if not self.result.passed: return self.result self.log(u"Checking required parameters for job") job_parameters = gf.config_xml_to_dict(contents, self.result, parse_job=True) self._check_required_parameters(self.XML_JOB_REQUIRED_PARAMETERS, job_parameters) if not self.result.passed: return self.result self.log(u"Checking required parameters for task") tasks_parameters = gf.config_xml_to_dict(contents, self.result, parse_job=False) for parameters in tasks_parameters: self.log([u"Checking required parameters for task: '%s'", parameters]) self._check_required_parameters(self.XML_TASK_REQUIRED_PARAMETERS, parameters) if not self.result.passed: return self.result return self.result
def test_config_xml_to_dict_task(self): tests = [ (None, []), (u"", []), (u"<job></job>", []), (u"<job><k1>v1</k1></job>", []), (u"<job><k1>v1</k1><k2></k2></job>", []), (u"<job><tasks></tasks></job>", []), (u"<job><tasks><foo></foo></tasks></job>", []), (u"<job><tasks><task></task></tasks></job>", [{}]), (u"<job><tasks><task></task><foo></foo></tasks></job>", [{}]), (u"<job><tasks><task></task><foo></foo><task></task></tasks></job>", [{}, {}]), (u"<job><tasks><task><k1></k1></task><foo></foo></tasks></job>", [{}]), (u"<job><tasks><task><k1>v1</k1></task></tasks></job>", [{u"k1": u"v1"}]), (u"<job><tasks><task><k1>v1</k1><k2>v2</k2></task></tasks></job>", [{u"k1": u"v1", u"k2": u"v2"}]), (u"<job><tasks><task><k1>v1</k1><k2> v2</k2></task></tasks></job>", [{u"k1": u"v1", u"k2": u"v2"}]), (u"<job><tasks><task><k1>v1</k1><k2> v2 </k2></task></tasks></job>", [{u"k1": u"v1", u"k2": u"v2"}]), (u"<job><tasks><task><k1>v1</k1><k2>v2 </k2></task></tasks></job>", [{u"k1": u"v1", u"k2": u"v2"}]), ( u"<job><tasks><task><k1>v1</k1></task><task><k2>v2</k2></task></tasks></job>", [{u"k1": u"v1"}, {u"k2": u"v2"}], ), ( u"<job><tasks><task><k1>v1</k1></task><task><k2>v2</k2></task><task></task></tasks></job>", [{u"k1": u"v1"}, {u"k2": u"v2"}, {}], ), ] for test in tests: self.assertEqual(gf.config_xml_to_dict(test[0], result=None, parse_job=False), test[1])
def test_config_xml_to_dict_job(self): tests = [ (None, {}), (u"", {}), (u"<job></job>", {}), (u"<job><k1>v1</k1></job>", { u"k1": u"v1" }), (u"<job><k1>v1</k1><k2></k2></job>", { u"k1": u"v1" }), (u"<job><k1>v1</k1><k2> </k2></job>", { u"k1": u"v1" }), (u"<job><k1>v1</k1><k2>v2</k2></job>", { u"k1": u"v1", u"k2": u"v2" }), (u"<job><k1>v1</k1><k2> v2</k2></job>", { u"k1": u"v1", u"k2": u"v2" }), (u"<job><k1>v1</k1><k2> v2 </k2></job>", { u"k1": u"v1", u"k2": u"v2" }), (u"<job><k1>v1</k1><k2>v2 </k2></job>", { u"k1": u"v1", u"k2": u"v2" }), ] for test in tests: self.assertEqual( gf.config_xml_to_dict(test[0], result=None, parse_job=True), test[1])
def test_config_xml_to_dict_task(self): tests = [ (None, []), (u"", []), (u"<job></job>", []), (u"<job><k1>v1</k1></job>", []), (u"<job><k1>v1</k1><k2></k2></job>", []), (u"<job><tasks></tasks></job>", []), (u"<job><tasks><foo></foo></tasks></job>", []), (u"<job><tasks><task></task></tasks></job>", [{}]), (u"<job><tasks><task></task><foo></foo></tasks></job>", [{}]), (u"<job><tasks><task></task><foo></foo><task></task></tasks></job>", [{}, {}]), (u"<job><tasks><task><k1></k1></task><foo></foo></tasks></job>", [{}]), (u"<job><tasks><task><k1>v1</k1></task></tasks></job>", [{ u"k1": u"v1" }]), (u"<job><tasks><task><k1>v1</k1><k2>v2</k2></task></tasks></job>", [{ u"k1": u"v1", u"k2": u"v2" }]), (u"<job><tasks><task><k1>v1</k1><k2> v2</k2></task></tasks></job>", [{ u"k1": u"v1", u"k2": u"v2" }]), (u"<job><tasks><task><k1>v1</k1><k2> v2 </k2></task></tasks></job>", [{ u"k1": u"v1", u"k2": u"v2" }]), (u"<job><tasks><task><k1>v1</k1><k2>v2 </k2></task></tasks></job>", [{ u"k1": u"v1", u"k2": u"v2" }]), (u"<job><tasks><task><k1>v1</k1></task><task><k2>v2</k2></task></tasks></job>", [{ u"k1": u"v1" }, { u"k2": u"v2" }]), (u"<job><tasks><task><k1>v1</k1></task><task><k2>v2</k2></task><task></task></tasks></job>", [{ u"k1": u"v1" }, { u"k2": u"v2" }, {}]), ] for test in tests: self.assertEqual( gf.config_xml_to_dict(test[0], result=None, parse_job=False), test[1])
def test_config_xml_to_dict_job(self): tests = [ (None, {}), (u"", {}), (u"<job></job>", {}), (u"<job><k1>v1</k1></job>", {u"k1": u"v1"}), (u"<job><k1>v1</k1><k2></k2></job>", {u"k1": u"v1"}), (u"<job><k1>v1</k1><k2> </k2></job>", {u"k1": u"v1"}), (u"<job><k1>v1</k1><k2>v2</k2></job>", {u"k1": u"v1", u"k2": u"v2"}), (u"<job><k1>v1</k1><k2> v2</k2></job>", {u"k1": u"v1", u"k2": u"v2"}), (u"<job><k1>v1</k1><k2> v2 </k2></job>", {u"k1": u"v1", u"k2": u"v2"}), (u"<job><k1>v1</k1><k2>v2 </k2></job>", {u"k1": u"v1", u"k2": u"v2"}), ] for test in tests: self.assertEqual(gf.config_xml_to_dict(test[0], result=None, parse_job=True), test[1])
def _analyze_xml_config(self, config_contents=None): """ Analyze the given container and return the corresponding job. If ``config_contents`` is ``None``, try reading it from the XML config file inside the container. :param string config_contents: the contents of the XML config file :rtype: :class:`~aeneas.job.Job` """ self.log(u"Analyzing container with XML config string") if config_contents is None: self.log(u"Analyzing container with XML config file") config_entry = self.container.entry_config_xml self.log([u"Found XML config entry '%s'", config_entry]) config_dir = os.path.dirname(config_entry) self.log([u"Directory of XML config entry: '%s'", config_dir]) self.log([u"Reading XML config entry: '%s'", config_entry]) config_contents = self.container.read_entry(config_entry) else: self.log(u"Analyzing container with XML config contents") config_dir = "" self.log(u"Converting config contents into job config dict") job_parameters = gf.config_xml_to_dict( config_contents, result=None, parse_job=True ) self.log(u"Converting config contents into tasks config dict") tasks_parameters = gf.config_xml_to_dict( config_contents, result=None, parse_job=False ) self.log(u"Calculating the path of the sync map root directory") sync_map_root_directory = gf.norm_join( config_dir, job_parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX] ) job_os_hierarchy_type = job_parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE] self.log([u"Path of the sync map root directory: '%s'", sync_map_root_directory]) self.log(u"Converting job config dict into job config string") config_string = gf.config_dict_to_string(job_parameters) job = Job(config_string) for task_parameters in tasks_parameters: self.log(u"Converting task config dict into task config string") config_string = gf.config_dict_to_string(task_parameters) self.log([u"Creating task with config string '%s'", config_string]) try: custom_id = task_parameters[gc.PPN_TASK_CUSTOM_ID] except KeyError: custom_id = "" task_info = [ custom_id, gf.norm_join( config_dir, task_parameters[gc.PPN_TASK_IS_TEXT_FILE_XML] ), gf.norm_join( config_dir, task_parameters[gc.PPN_TASK_IS_AUDIO_FILE_XML] ) ] self.log([u"Creating task: '%s'", str(task_info)]) task = self._create_task( task_info, config_string, sync_map_root_directory, job_os_hierarchy_type ) job.add_task(task) return job
def _analyze_xml_config(self, config_contents=None): """ Analyze the given container and return the corresponding job. If ``config_contents`` is ``None``, try reading it from the XML config file inside the container. :param config_contents: the contents of the XML config file :type config_contents: string :rtype: :class:`aeneas.job.Job` """ # TODO break this function down into smaller functions self._log("Analyzing container with XML config string") if config_contents is None: self._log("Analyzing container with XML config file") config_entry = self.container.entry_config_xml self._log(["Found XML config entry '%s'", config_entry]) config_dir = os.path.dirname(config_entry) self._log(["Directory of XML config entry: '%s'", config_dir]) self._log(["Reading XML config entry: '%s'", config_entry]) config_contents = self.container.read_entry(config_entry) else: self._log("Analyzing container with XML config contents") config_dir = "" # remove BOM #self._log("Removing BOM") #config_contents = gf.remove_bom(config_contents) # get the job parameters and tasks parameters self._log("Converting config contents into job config dict") job_parameters = gf.config_xml_to_dict( config_contents, result=None, parse_job=True ) self._log("Converting config contents into tasks config dict") tasks_parameters = gf.config_xml_to_dict( config_contents, result=None, parse_job=False ) # compute the root directory for the sync map files self._log("Calculating the path of the sync map root directory") sync_map_root_directory = gf.norm_join( config_dir, job_parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX] ) job_os_hierarchy_type = job_parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE] self._log(["Path of the sync map root directory: '%s'", sync_map_root_directory]) # create the Job object to be returned self._log("Converting job config dict into job config string") config_string = gf.config_dict_to_string(job_parameters) job = Job(config_string) # create the Task objects for task_parameters in tasks_parameters: self._log("Converting task config dict into task config string") config_string = gf.config_dict_to_string(task_parameters) self._log(["Creating task with config string '%s'", config_string]) try: custom_id = task_parameters[gc.PPN_TASK_CUSTOM_ID] except KeyError: custom_id = "" task_info = [ custom_id, gf.norm_join( config_dir, task_parameters[gc.PPN_TASK_IS_TEXT_FILE_XML] ), gf.norm_join( config_dir, task_parameters[gc.PPN_TASK_IS_AUDIO_FILE_XML] ) ] self._log(["Creating task: '%s'", str(task_info)]) task = self._create_task( task_info, config_string, sync_map_root_directory, job_os_hierarchy_type ) job.add_task(task) # return the Job return job
def check_contents_xml_config_file(self, config_contents): """ Check whether the given XML config contents is well formed and contains all the requested parameters. :param config_contents: :type config_contents: string :rtype: :class:`aeneas.validator.ValidatorResult` """ self._log("Checking contents XML config file") result = ValidatorResult() # remove BOM, if any #self._log("Removing BOM") #config_contents = gf.remove_bom(config_contents) # check if it is well encoded result = self.check_string_well_encoded(config_contents) if not result.passed: self._log("Failed") return result # check required parameters for the job section self._log("Checking required parameters for job") required_parameters = [ gc.PPN_JOB_OS_FILE_NAME, gc.PPN_JOB_OS_CONTAINER_FORMAT, gc.PPN_JOB_OS_HIERARCHY_TYPE, gc.PPN_JOB_OS_HIERARCHY_PREFIX ] job_parameters = gf.config_xml_to_dict( config_contents, result, parse_job=True ) self._check_required_parameters( required_parameters, job_parameters, result ) if not result.passed: self._log("Failed") return result # check required parameters for each task section self._log("Checking required parameters for task") required_parameters = [ gc.PPN_TASK_LANGUAGE, gc.PPN_TASK_IS_TEXT_FILE_FORMAT, gc.PPN_TASK_IS_TEXT_FILE_XML, gc.PPN_TASK_IS_AUDIO_FILE_XML, gc.PPN_TASK_OS_FILE_NAME, gc.PPN_TASK_OS_FILE_FORMAT, ] tasks_parameters = gf.config_xml_to_dict( config_contents, result, parse_job=False ) for parameters in tasks_parameters: self._log(["Checking required parameters for task: '%s'", parameters]) self._check_required_parameters( required_parameters, parameters, result ) if not result.passed: self._log("Failed") return result # return result self._log(["Checking contents XML config file: returning %s", result.passed]) return result
def check_contents_xml_config_file(self, config_contents): """ Check whether the given XML config contents is well formed and contains all the requested parameters. :param config_contents: :type config_contents: string :rtype: :class:`aeneas.validator.ValidatorResult` """ self._log("Checking contents XML config file") result = ValidatorResult() # remove BOM, if any #self._log("Removing BOM") #config_contents = gf.remove_bom(config_contents) # check if it is well encoded result = self.check_string_well_encoded(config_contents) if not result.passed: self._log("Failed") return result # check required parameters for the job section self._log("Checking required parameters for job") required_parameters = [ gc.PPN_JOB_OS_FILE_NAME, gc.PPN_JOB_OS_CONTAINER_FORMAT, gc.PPN_JOB_OS_HIERARCHY_TYPE, gc.PPN_JOB_OS_HIERARCHY_PREFIX ] job_parameters = gf.config_xml_to_dict(config_contents, result, parse_job=True) self._check_required_parameters(required_parameters, job_parameters, result) if not result.passed: self._log("Failed") return result # check required parameters for each task section self._log("Checking required parameters for task") required_parameters = [ gc.PPN_TASK_LANGUAGE, gc.PPN_TASK_IS_TEXT_FILE_FORMAT, gc.PPN_TASK_IS_TEXT_FILE_XML, gc.PPN_TASK_IS_AUDIO_FILE_XML, gc.PPN_TASK_OS_FILE_NAME, gc.PPN_TASK_OS_FILE_FORMAT, ] tasks_parameters = gf.config_xml_to_dict(config_contents, result, parse_job=False) for parameters in tasks_parameters: self._log( ["Checking required parameters for task: '%s'", parameters]) self._check_required_parameters(required_parameters, parameters, result) if not result.passed: self._log("Failed") return result # return result self._log( ["Checking contents XML config file: returning %s", result.passed]) return result
def _analyze_xml_config(self, config_contents=None): """ Analyze the given container and return the corresponding job. If ``config_contents`` is ``None``, try reading it from the XML config file inside the container. :param string config_contents: the contents of the XML config file :rtype: :class:`~aeneas.job.Job` """ self.log(u"Analyzing container with XML config string") if config_contents is None: self.log(u"Analyzing container with XML config file") config_entry = self.container.entry_config_xml self.log([u"Found XML config entry '%s'", config_entry]) config_dir = os.path.dirname(config_entry) self.log([u"Directory of XML config entry: '%s'", config_dir]) self.log([u"Reading XML config entry: '%s'", config_entry]) config_contents = self.container.read_entry(config_entry) else: self.log(u"Analyzing container with XML config contents") config_dir = "" self.log(u"Converting config contents into job config dict") job_parameters = gf.config_xml_to_dict(config_contents, result=None, parse_job=True) self.log(u"Converting config contents into tasks config dict") tasks_parameters = gf.config_xml_to_dict(config_contents, result=None, parse_job=False) self.log(u"Calculating the path of the sync map root directory") sync_map_root_directory = gf.norm_join( config_dir, job_parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX]) job_os_hierarchy_type = job_parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE] self.log([ u"Path of the sync map root directory: '%s'", sync_map_root_directory ]) self.log(u"Converting job config dict into job config string") config_string = gf.config_dict_to_string(job_parameters) job = Job(config_string) for task_parameters in tasks_parameters: self.log(u"Converting task config dict into task config string") config_string = gf.config_dict_to_string(task_parameters) self.log([u"Creating task with config string '%s'", config_string]) try: custom_id = task_parameters[gc.PPN_TASK_CUSTOM_ID] except KeyError: custom_id = "" task_info = [ custom_id, gf.norm_join(config_dir, task_parameters[gc.PPN_TASK_IS_TEXT_FILE_XML]), gf.norm_join(config_dir, task_parameters[gc.PPN_TASK_IS_AUDIO_FILE_XML]) ] self.log([u"Creating task: '%s'", str(task_info)]) task = self._create_task(task_info, config_string, sync_map_root_directory, job_os_hierarchy_type) job.add_task(task) return job
def _analyze_xml_config(self, config_contents=None): """ Analyze the given container and return the corresponding job. If ``config_contents`` is ``None``, try reading it from the XML config file inside the container. :param config_contents: the contents of the XML config file :type config_contents: string :rtype: :class:`aeneas.job.Job` """ # TODO break this function down into smaller functions self._log("Analyzing container with XML config string") if config_contents is None: self._log("Analyzing container with XML config file") config_entry = self.container.entry_config_xml self._log(["Found XML config entry '%s'", config_entry]) config_dir = os.path.dirname(config_entry) self._log(["Directory of XML config entry: '%s'", config_dir]) self._log(["Reading XML config entry: '%s'", config_entry]) config_contents = self.container.read_entry(config_entry) else: self._log("Analyzing container with XML config contents") config_dir = "" # remove BOM #self._log("Removing BOM") #config_contents = gf.remove_bom(config_contents) # get the job parameters and tasks parameters self._log("Converting config contents into job config dict") job_parameters = gf.config_xml_to_dict(config_contents, result=None, parse_job=True) self._log("Converting config contents into tasks config dict") tasks_parameters = gf.config_xml_to_dict(config_contents, result=None, parse_job=False) # compute the root directory for the sync map files self._log("Calculating the path of the sync map root directory") sync_map_root_directory = gf.norm_join( config_dir, job_parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX]) job_os_hierarchy_type = job_parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE] self._log([ "Path of the sync map root directory: '%s'", sync_map_root_directory ]) # create the Job object to be returned self._log("Converting job config dict into job config string") config_string = gf.config_dict_to_string(job_parameters) job = Job(config_string) # create the Task objects for task_parameters in tasks_parameters: self._log("Converting task config dict into task config string") config_string = gf.config_dict_to_string(task_parameters) self._log(["Creating task with config string '%s'", config_string]) try: custom_id = task_parameters[gc.PPN_TASK_CUSTOM_ID] except KeyError: custom_id = "" task_info = [ custom_id, gf.norm_join(config_dir, task_parameters[gc.PPN_TASK_IS_TEXT_FILE_XML]), gf.norm_join(config_dir, task_parameters[gc.PPN_TASK_IS_AUDIO_FILE_XML]) ] self._log(["Creating task: '%s'", str(task_info)]) task = self._create_task(task_info, config_string, sync_map_root_directory, job_os_hierarchy_type) job.add_task(task) # return the Job return job