def test_update_json_list_in_file_two_times(self):
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")

        param1 = computation_job_param.ComputationJobParam("minlat", "float", "par title", "min latitude")
        param2 = computation_job_param.ComputationJobParam("maxlat", "float", "par title", "max latitude")
        
        inputs = computation_job_inputs.ComputationJobInputs()
        
        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("minlat", target_json_file , "['Config']['latLim'][0]", source_template_json_file)
        #action2 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("maxlat", target_json_file , "['Config']['latLim'][1]", source_template_json_file)

        inputs.add_input ( param1 )
        inputs.add_input ( param2 )

        inputs.set_values( { "minlat" : "75.5", "maxlat" : "76.5" } )
        action1.set_inputs( inputs )

        #check target_json_file
        json_text = target_json_file.text()
        j=json.loads(json_text)
        self.assertTrue( 75.5, j['Config']['latLim'][0] )
        self.assertTrue( 76.5, j['Config']['latLim'][1] )
    def test_update_json_list_in_file_two_times(self):
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(
            r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
        )

        param1 = computation_job_param.ComputationJobParam(
            "minlat", "float", "par title", "min latitude")
        param2 = computation_job_param.ComputationJobParam(
            "maxlat", "float", "par title", "max latitude")

        inputs = computation_job_inputs.ComputationJobInputs()

        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
            "minlat", target_json_file, "['Config']['latLim'][0]",
            source_template_json_file)

        inputs.add_input(param1)
        inputs.add_input(param2)

        inputs.set_values({"minlat": "75.5", "maxlat": "76.5"})
        action1.set_inputs(inputs)

        # check target_json_file
        json_text = target_json_file.text()
        j = json.loads(json_text)
        self.assertTrue(75.5, j['Config']['latLim'][0])
        self.assertTrue(76.5, j['Config']['latLim'][1])
    def test_create_2_json_file_action(self):
        p1 = computation_job_param.ComputationJobParam("mypar1",
                                                       "application/json",
                                                       "par 1",
                                                       "par descr 1",
                                                       max_occurencies=2)
        inputs = computation_job_inputs.ComputationJobInputs()
        inputs.add_input(p1)

        actions = computational_job_input_actions.ComputationalJobInputActions(
        )
        a1 = computational_job_input_action_create_json_file. \
            ComputationalJobInputActionCreateJSONFile("mypar1",
                                                      path.path("./json_out_${json_path_expr}.json"),
                                                      "['Asset']['id']",
                                                      path.path("./src/wpsremote/xmpp_data/test/asset_schema.json"))
        actions.add_actions(a1)

        inputs.set_values({
            "mypar1": [
                TestComputationJobInputs.json_text1,
                TestComputationJobInputs.json_text2
            ]
        })
        actions.execute(inputs)

        self.assertTrue(a1.exists())
    def test_update_json_file_action_with_string(self):
        os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                              '..')))
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(
            r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
        )

        param1 = computation_job_param.ComputationJobParam(
            "path_file_name", "string", "par 1", "path_file_name descr")
        inputs = computation_job_inputs.ComputationJobInputs()
        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
            "path_file_name", target_json_file, "['Config']['pathFilename']",
            source_template_json_file)

        inputs.add_input(param1)
        inputs.set_values({"path_file_name": "thisIsOK.txt"})
        action1.set_inputs(inputs)

        # check target_json_file
        json_text = target_json_file.text()
        j = json.loads(json_text)
        self.assertTrue("thisIsOK.txt", j['Config']['pathFilename'])
 def test_FtpUpload(self, mock_ftp):
     mockFTP = mock_ftp.return_value
     try:
         ftpUpload = FtpUpload("host", "username", "password")
         ftpUpload.ftp = mockFTP
         ftpUpload.setHost("host:port", "username", "password")
         ftpUpload.setMd5File("./src/wpsremote/xmpp_data/test/test_upload")
         ftpUpload.Upload()
         ftpUpload.deleteOldFiles()
         ftpUpload.finish()
         path("./src/wpsremote/xmpp_data/test/test_upload").remove()
     except Exception as e:
         self.fail(e)
 def test_copyfile(self):
     os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                           '..')))
     target_json_file = path.path("./copy_of_source_json.json")
     if target_json_file.exists():
         target_json_file.remove()
     source_template_json_file = path.path(
         r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
     )
     action = computational_job_input_action_copyfile.ComputationalJobInputActionCopyFile(
         source=source_template_json_file, target=target_json_file)
     action.set_inputs("test")
     self.assertTrue(target_json_file.exists())
     target_json_text = target_json_file.text()
     source_json_text = source_template_json_file.text()
     self.assertEqual(target_json_text, source_json_text)
     target_json_file.remove()
    def test_update_json_file_action_with_string(self):
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")

        param1 = computation_job_param.ComputationJobParam("path_file_name", "string", "par 1", "path_file_name descr")
        inputs = computation_job_inputs.ComputationJobInputs()
        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("path_file_name", target_json_file , "['Config']['pathFilename']", source_template_json_file)

        inputs.add_input ( param1 )
        inputs.set_values( { "path_file_name" : "thisIsOK.txt" } )
        action1.set_inputs( inputs )

        #check target_json_file
        json_text = target_json_file.text()
        j=json.loads(json_text)
        self.assertTrue( "thisIsOK.txt", j['Config']['pathFilename'] )
    def test_update_json_file_action_with_int(self):
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")

        param1 = computation_job_param.ComputationJobParam("numberOfevaluations", "int", "par 1", "numberOfevaluations descr")
        inputs = computation_job_inputs.ComputationJobInputs()
        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("numberOfevaluations", target_json_file , "['Config']['nEvaluations']", source_template_json_file)

        inputs.add_input ( param1 )
        inputs.set_values( { "numberOfevaluations" : "100" } )
        action1.set_inputs( inputs )

        #check target_json_file
        json_text = target_json_file.text()
        j=json.loads(json_text)
        self.assertTrue( 100, j['Config']['nEvaluations'] )
    def test_create_json_file_action(self):
        os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                              '..')))
        p1 = computation_job_param.ComputationJobParam("mypar1",
                                                       "application/json",
                                                       "par 1", "par descr 1")
        inputs = computation_job_inputs.ComputationJobInputs()
        inputs.add_input(p1)

        actions = computational_job_input_actions.ComputationalJobInputActions(
        )
        a1 = computational_job_input_action_create_json_file. \
            ComputationalJobInputActionCreateJSONFile("mypar1",
                                                      path.path("./json_out_${json_path_expr}.json"),
                                                      "['Asset']['id']",
                                                      path.path("./src/wpsremote/xmpp_data/test/asset_schema.json"))
        actions.add_actions(a1)

        inputs.set_values({"mypar1": TestComputationJobInputs.json_text1})
        actions.execute(inputs)

        self.assertTrue(a1.exists())
    def test_update_json_file_action_with_int(self):
        target_json_file = path.path("./json_out.json")
        if target_json_file.exists():
            target_json_file.remove()
        source_template_json_file = path.path(
            r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
        )

        param1 = computation_job_param.ComputationJobParam(
            "numberOfevaluations", "int", "par 1", "numberOfevaluations descr")
        inputs = computation_job_inputs.ComputationJobInputs()
        action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
            "numberOfevaluations", target_json_file,
            "['Config']['nEvaluations']", source_template_json_file)

        inputs.add_input(param1)
        inputs.set_values({"numberOfevaluations": "100"})
        action1.set_inputs(inputs)

        # check target_json_file
        json_text = target_json_file.text()
        j = json.loads(json_text)
        self.assertTrue(100, j['Config']['nEvaluations'])
 def test_instance_methods(self):
     rc = Resource()
     self.assertIsNone(rc.start_time())
     self.assertIsNone(rc.cmd_line())
     self.assertIsNone(rc.unique_id())
     self.assertIsNone(rc.set_unique_id(1))
     self.assertIsNone(rc.sendbox_path())
     self.assertIsNone(rc.processbot_pid())
     self.assertIsNone(rc.set_processbot_pid(1))
     self.assertIsNone(rc.spawned_process_pids())
     self.assertIsNone(rc.spawned_process_cmd())
     sandbox_root = path("./src/wpsremote/xmpp_data/test/test_dir")
     unique_id = str(random.randint(1, 1000)).zfill(5)
     sendbox_path = sandbox_root / str(unique_id)
     rc.set_from_servicebot(unique_id, sendbox_path)
     self.assertEqual(rc.sendbox_path(), sendbox_path)
     self.assertEqual(rc._unique_id, unique_id)
     rc.set_from_processbot(os.getpid(), [unique_id])
     self.assertIn(unique_id, rc._spawned_process_pids)
     try:
         rc.read()
     except Exception as e:
         self.fail(e)
    def test_create_json_file_action(self):
        p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", "par descr 1")
        inputs = computation_job_inputs.ComputationJobInputs()
        inputs.add_input( p1 )

        actions = computational_job_input_actions.ComputationalJobInputActions()
        a1 = computational_job_input_action_create_json_file.ComputationalJobInputActionCreateJSONFile("mypar1", path.path("./json_out_${json_path_expr}.json"), "['Asset']['id']", path.path("./src/wpsremote/xmpp_data/test/asset_schema.json")) 
        actions.add_actions( a1 )
        
        
        inputs.set_values( { "mypar1" : TestComputationJobInputs.json_text1 } )
        actions.execute( inputs )

        self.assertTrue( a1.exists() )
Beispiel #13
0
 def test_path_methods(self):
     p = path.path("test/path")
     self.assertEqual(p.__repr__(), "path('test/path')")
     self.assertEqual(p.__add__("/add"), "test/path/add")
     self.assertEqual(p.__radd__("radd/"), "radd/test/path")
     self.assertEqual(p.__div__("rel"), "test/path/rel")
     self.assertEqual(p.__truediv__("rel"), "test/path/rel")
     self.assertFalse(p.isabs())
     self.assertEqual(p.basename(), "path")
     self.assertEqual(p.name, "path")
     self.assertEqual(p.normcase(), "test/path")
     self.assertEqual(p.normpath(), "test/path")
     self.assertEqual(p.expanduser(), "test/path")
     self.assertEqual(p.expandvars(), "test/path")
     self.assertEqual(p.dirname(), "test")
     self.assertEqual(p.parent, "test")
     self.assertEqual(p.expand(), "test/path")
     self.assertEqual(p._get_namebase(), "path")
     self.assertEqual(p.namebase, "path")
     self.assertEqual(p._get_ext(), "")
     self.assertEqual(p.ext, "")
     self.assertEqual(p._get_drive(), "")
     self.assertEqual(p.drive, "")
     self.assertEqual(p.splitpath(), (path.path('test'), 'path'))
     self.assertEqual(p.splitdrive(),
                      (path.path(''), path.path('test/path')))
     self.assertEqual(p.splitext(), (path.path('test/path'), ''))
     self.assertEqual(p.stripext(), "test/path")
     self.assertEqual(p.joinpath("join", "path"), "test/path/join/path")
     self.assertEqual(p.splitall(), [path.path(''), 'test', 'path'])
     self.assertEqual(p.relpath(), "test/path")
     self.assertEqual(p.relpathto("dest"), "../../dest")
     with self.assertRaises(OSError):
         p.listdir(pattern=None)
     existing_path = path.path("./src/wpsremote/xmpp_data/test")
     self.assertEqual(len(existing_path.listdir(pattern=None)), 13)
     for d in existing_path.listdir(pattern=None):
         self.assertIn(d, TestPath.LIST_DIRS)
     self.assertEqual(len(existing_path.dirs(pattern=None)), 2)
     for d in existing_path.dirs(pattern=None):
         self.assertIn(d, TestPath.DIRS)
     self.assertEqual(len(existing_path.files()), 11)
     for f in existing_path.files():
         self.assertIn(f, TestPath.LIST_DIRS)
     self.assertTrue(existing_path.fnmatch("*test*"))
     self.assertEqual(
         existing_path.glob("*test_dir*"),
         [path.path('./src/wpsremote/xmpp_data/test/test_dir')])
     existing_file = path.path("./src/wpsremote/xmpp_data/test/test_file")
     if sys.version_info[0] < 3:
         self.assertIsInstance(existing_file.open(), file)
     else:
         from io import IOBase
         self.assertIsInstance(existing_file.open(), IOBase)
     self.assertEqual(existing_file.bytes(), "test content")
     existing_file.write_bytes("test file to write")
     self.assertEqual(existing_file.bytes(), "test file to write")
     self.assertEqual(existing_file.text(), "test file to write")
     existing_file.write_text("test content")
     self.assertEqual(existing_file.text(), "test content")
     existing_file.write_lines(["line 1", "line 2"])
     self.assertEqual(existing_file.lines(), ["line 1\n", "line 2\n"])
     existing_file.write_text("test content")
     # Methods for querying the filesystem
     self.assertTrue(existing_file.exists())
     self.assertFalse(path.path("./not/existing/path/test_file").exists())
     self.assertTrue(existing_path.isdir())
     self.assertTrue(existing_file.isfile())
     self.assertTrue(path.path("/").ismount())
     # mkdir
     new_dir = path.path('./src/wpsremote/xmpp_data/test/new_dir')
     new_dir.mkdir()
     self.assertTrue(new_dir.exists())
     new_dirs = path.path('./src/wpsremote/xmpp_data/test/new_dirs/new_dir')
     new_dirs.makedirs()
     self.assertTrue(new_dirs.exists())
     # rmdir
     new_dir.rmdir()
     self.assertFalse(new_dir.exists())
     new_dirs.removedirs()
     self.assertFalse(new_dirs.exists())
     # touch
     touch_file = path.path('./src/wpsremote/xmpp_data/test/new_file')
     self.assertFalse(touch_file.exists())
     touch_file.touch()
     self.assertTrue(touch_file.exists())
     # remove
     touch_file.remove()
     self.assertFalse(touch_file.exists())
     # unlink
     touch_file.touch()
     self.assertTrue(touch_file.exists())
     touch_file.unlink()
     self.assertFalse(touch_file.exists())
     # copyfile
     existing_file.copy('./src/wpsremote/xmpp_data/test/copy_of_test_file')
     copied_file = path.path(
         './src/wpsremote/xmpp_data/test/copy_of_test_file')
     self.assertTrue(copied_file.exists())
     copied_file.remove()
Beispiel #14
0
 def test_process_bot(self):
     wps_agent = MockWPSAgentProcess(self.args)
     # test bot
     process_bot = wps_agent.create_bot()
     self.assertIsInstance(process_bot, ProcessBot)
     self.assertTrue(process_bot._active)
     self.assertFalse(process_bot._finished)
     self.assertIsInstance(process_bot._input_parameters_defs, ComputationJobInputs)
     self.assertIsInstance(process_bot._input_params_actions, ComputationalJobInputActions)
     self.assertEqual(process_bot._input_values, {"test_message": "test message"})
     self.assertEqual(process_bot._max_running_time, datetime.timedelta(0, 10))
     self.assertEqual(process_bot.max_execution_time(), datetime.timedelta(0, 10))
     self.assertIsInstance(process_bot._output_parameters_defs, OutputParameters)
     self.assertEqual(process_bot._remote_wps_baseurl, "test_base_url")
     self.assertEqual(
         process_bot._resource_file_dir,
         path("./src/wpsremote/xmpp_data/test/resource_dir")
     )
     self.assertEqual(
         process_bot.get_resource_file_dir(),
         path("./src/wpsremote/xmpp_data/test/resource_dir")
     )
     self.assertEqual(len(process_bot._stdout_action), 6)
     for stdout_action in process_bot._stdout_action:
         self.assertIn(stdout_action, ['ignore', 'progress', 'log', 'log', 'log', 'abort'])
     self.assertEqual(len(process_bot._stdout_parser), 6)
     for stdout_parser in process_bot._stdout_parser:
         self.assertIn(stdout_parser, [
             '.*\\[DEBUG\\](.*)',
             '.*\\[INFO\\] ProgressInfo\\:([-+]?[0-9]*\\.?[0-9]*)\\%',
             '.*\\[(INFO)\\](.*)',
             '.*\\[(WARN)\\](.*)',
             '.*\\[(ERROR)\\](.*)',
             '.*\\[(CRITICAL)\\](.*)'
     ])
     self.assertEqual(process_bot._uniqueExeId, "123")
     self.assertEqual(process_bot._uploader, None)
     self.assertEqual(process_bot._wps_execution_shared_dir, None)
     self.assertEqual(process_bot.get_wps_execution_shared_dir(), None)
     self.assertEqual(process_bot.description, "foo service")
     self.assertEqual(process_bot.namespace, "default")
     self.assertEqual(process_bot.service, "Service")
     self.assertEqual(
         process_bot.workdir(),
         process_bot._output_dir + "/" + process_bot._uniqueExeId
     )
     # handle_finish
     finish_msg = {
         "type": "normal",
         "body": "topic=finish",
         "from": self.from_obj_mock
     }
     finished_message = self.xmpp_bus.CreateIndipendentMessage(finish_msg)
     with self.assertRaises(SystemExit) as cm_finish:
         process_bot.handle_finish(finished_message)
         self.assertEqual(cm_finish.exception.code, 0)
     # handle_abort
     abort_msg = {
         "type": "normal",
         "body": "topic=abort",
         "from": self.from_obj_mock
     }
     abort_message = self.xmpp_bus.CreateIndipendentMessage(abort_msg)
     with self.assertRaises(SystemExit) as cm_abort:
         process_bot.handle_abort(abort_message)
         self.assertEqual(cm_abort.exception.code, -1)
     with self.assertRaises(SystemExit):
         process_bot.exit(0)
     # kill process
     self.cleaner.kill_processbot()
     self.assertFalse(psutil.pid_exists(self.cleaner._processbot_pid))
Beispiel #15
0
 def test_service_bot(self):
     wps_agent = MockWPSAgentService(self.args)
     # test bot
     service_bot = wps_agent.create_bot()
     self.assertIsInstance(service_bot, ServiceBot)
     self.assertTrue(service_bot._active)
     self.assertIsInstance(service_bot._input_parameters_defs, ComputationJobInputs)
     self.assertEqual(service_bot._max_running_time, datetime.timedelta(0, 10))
     self.assertEqual(
         service_bot._output_dir,
         path('src\\wpsremote\\xmpp_data\\test\\tmp')
     )
     self.assertIsInstance(service_bot._output_parameters_defs, OutputParameters)
     self.assertTrue(service_bot._redirect_process_stdout_to_logger)
     self.assertEqual(
         service_bot._remote_config_filepath,
         './src/wpsremote/xmpp_data/test/test_remote.config'
     )
     self.assertIsNone(service_bot._remote_wps_endpoint)
     self.assertEqual(
         service_bot._resource_file_dir,
         path('./src/wpsremote/xmpp_data/test/resource_dir')
     )
     self.assertEqual(
         service_bot.get_resource_file_dir(),
         path("./src/wpsremote/xmpp_data/test/resource_dir")
     )
     self.assertIsInstance(service_bot._resource_monitor, MockResourceMonitor)
     self.assertEqual(
         service_bot._service_config_file,
         './src/wpsremote/xmpp_data/test/test_service.config'
     )
     self.assertIsNone(service_bot._wps_execution_shared_dir)
     self.assertEqual(
         service_bot.max_execution_time(),
         datetime.timedelta(0, 10)
     )
     self.assertEqual(
         service_bot.get_wps_execution_shared_dir(),
         None
     )
     self.assertIsNotNone(service_bot.bus)
     self.assertEqual(service_bot.description, 'foo service')
     self.assertEqual(service_bot.namespace, 'default')
     self.assertEqual(service_bot.running_process, {})
     self.assertEqual(service_bot.service, 'Service')
     self.assertEqual(
         service_bot._process_blacklist,
         json.loads(service_bot._remote_config.get("DEFAULT", "process_blacklist"))
     )
     self.assertEqual(
         resource_monitor.ResourceMonitor.capacity,
         service_bot._remote_config.getint("DEFAULT", "capacity")
     )
     self.assertEqual(
         resource_monitor.ResourceMonitor.load_threshold,
         service_bot._remote_config.getint("DEFAULT", "load_threshold")
     )
     self.assertEqual(
         resource_monitor.ResourceMonitor.load_average_scan_minutes,
         service_bot._remote_config.getint("DEFAULT", "load_average_scan_minutes")
     )
     PCk = None
     try:
         process_weight_class_name = service_bot._service_config.get("DEFAULT", "process_weight")
         PCk = introspection.get_class_no_arg(process_weight_class_name)
     except BaseException:
         PCk = None
     if not PCk:
         try:
             process_weight = json.loads(
                 service_bot._service_config.get("DEFAULT", "process_weight"))
         except BaseException:
             process_weight = {"weight": "0", "coefficient": "1.0"}
         PCk = resource_monitor.ProcessWeight(process_weight)
     self.assertEqual(PCk.__class__, resource_monitor.ProcessWeight)
     _request_load = PCk.request_weight(None)
     self.assertEqual(_request_load, 15.0)
     # removing tmp files
     params_file_path = path(self.params_file)
     params_file_path.remove()