def test_update_json_list_in_file_two_times(self):
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")
param1 = computation_job_param.ComputationJobParam("minlat", "float", "par title", "min latitude")
param2 = computation_job_param.ComputationJobParam("maxlat", "float", "par title", "max latitude")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("minlat", target_json_file , "['Config']['latLim'][0]", source_template_json_file)
#action2 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("maxlat", target_json_file , "['Config']['latLim'][1]", source_template_json_file)
inputs.add_input ( param1 )
inputs.add_input ( param2 )
inputs.set_values( { "minlat" : "75.5", "maxlat" : "76.5" } )
action1.set_inputs( inputs )
#check target_json_file
json_text = target_json_file.text()
j=json.loads(json_text)
self.assertTrue( 75.5, j['Config']['latLim'][0] )
self.assertTrue( 76.5, j['Config']['latLim'][1] )
def test_update_json_list_in_file_two_times(self):
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(
r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
)
param1 = computation_job_param.ComputationJobParam(
"minlat", "float", "par title", "min latitude")
param2 = computation_job_param.ComputationJobParam(
"maxlat", "float", "par title", "max latitude")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
"minlat", target_json_file, "['Config']['latLim'][0]",
source_template_json_file)
inputs.add_input(param1)
inputs.add_input(param2)
inputs.set_values({"minlat": "75.5", "maxlat": "76.5"})
action1.set_inputs(inputs)
# check target_json_file
json_text = target_json_file.text()
j = json.loads(json_text)
self.assertTrue(75.5, j['Config']['latLim'][0])
self.assertTrue(76.5, j['Config']['latLim'][1])
def test_create_2_json_file_action(self):
p1 = computation_job_param.ComputationJobParam("mypar1",
"application/json",
"par 1",
"par descr 1",
max_occurencies=2)
inputs = computation_job_inputs.ComputationJobInputs()
inputs.add_input(p1)
actions = computational_job_input_actions.ComputationalJobInputActions(
)
a1 = computational_job_input_action_create_json_file. \
ComputationalJobInputActionCreateJSONFile("mypar1",
path.path("./json_out_${json_path_expr}.json"),
"['Asset']['id']",
path.path("./src/wpsremote/xmpp_data/test/asset_schema.json"))
actions.add_actions(a1)
inputs.set_values({
"mypar1": [
TestComputationJobInputs.json_text1,
TestComputationJobInputs.json_text2
]
})
actions.execute(inputs)
self.assertTrue(a1.exists())
def test_update_json_file_action_with_string(self):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
'..')))
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(
r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
)
param1 = computation_job_param.ComputationJobParam(
"path_file_name", "string", "par 1", "path_file_name descr")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
"path_file_name", target_json_file, "['Config']['pathFilename']",
source_template_json_file)
inputs.add_input(param1)
inputs.set_values({"path_file_name": "thisIsOK.txt"})
action1.set_inputs(inputs)
# check target_json_file
json_text = target_json_file.text()
j = json.loads(json_text)
self.assertTrue("thisIsOK.txt", j['Config']['pathFilename'])
def test_FtpUpload(self, mock_ftp):
mockFTP = mock_ftp.return_value
try:
ftpUpload = FtpUpload("host", "username", "password")
ftpUpload.ftp = mockFTP
ftpUpload.setHost("host:port", "username", "password")
ftpUpload.setMd5File("./src/wpsremote/xmpp_data/test/test_upload")
ftpUpload.Upload()
ftpUpload.deleteOldFiles()
ftpUpload.finish()
path("./src/wpsremote/xmpp_data/test/test_upload").remove()
except Exception as e:
self.fail(e)
def test_copyfile(self):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
'..')))
target_json_file = path.path("./copy_of_source_json.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(
r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
)
action = computational_job_input_action_copyfile.ComputationalJobInputActionCopyFile(
source=source_template_json_file, target=target_json_file)
action.set_inputs("test")
self.assertTrue(target_json_file.exists())
target_json_text = target_json_file.text()
source_json_text = source_template_json_file.text()
self.assertEqual(target_json_text, source_json_text)
target_json_file.remove()
def test_update_json_file_action_with_string(self):
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")
param1 = computation_job_param.ComputationJobParam("path_file_name", "string", "par 1", "path_file_name descr")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("path_file_name", target_json_file , "['Config']['pathFilename']", source_template_json_file)
inputs.add_input ( param1 )
inputs.set_values( { "path_file_name" : "thisIsOK.txt" } )
action1.set_inputs( inputs )
#check target_json_file
json_text = target_json_file.text()
j=json.loads(json_text)
self.assertTrue( "thisIsOK.txt", j['Config']['pathFilename'] )
def test_update_json_file_action_with_int(self):
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json")
param1 = computation_job_param.ComputationJobParam("numberOfevaluations", "int", "par 1", "numberOfevaluations descr")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile("numberOfevaluations", target_json_file , "['Config']['nEvaluations']", source_template_json_file)
inputs.add_input ( param1 )
inputs.set_values( { "numberOfevaluations" : "100" } )
action1.set_inputs( inputs )
#check target_json_file
json_text = target_json_file.text()
j=json.loads(json_text)
self.assertTrue( 100, j['Config']['nEvaluations'] )
def test_create_json_file_action(self):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__),
'..')))
p1 = computation_job_param.ComputationJobParam("mypar1",
"application/json",
"par 1", "par descr 1")
inputs = computation_job_inputs.ComputationJobInputs()
inputs.add_input(p1)
actions = computational_job_input_actions.ComputationalJobInputActions(
)
a1 = computational_job_input_action_create_json_file. \
ComputationalJobInputActionCreateJSONFile("mypar1",
path.path("./json_out_${json_path_expr}.json"),
"['Asset']['id']",
path.path("./src/wpsremote/xmpp_data/test/asset_schema.json"))
actions.add_actions(a1)
inputs.set_values({"mypar1": TestComputationJobInputs.json_text1})
actions.execute(inputs)
self.assertTrue(a1.exists())
def test_update_json_file_action_with_int(self):
target_json_file = path.path("./json_out.json")
if target_json_file.exists():
target_json_file.remove()
source_template_json_file = path.path(
r"./src/wpsremote/xmpp_data/test/CMREOAA_MainConfigFile_template.json"
)
param1 = computation_job_param.ComputationJobParam(
"numberOfevaluations", "int", "par 1", "numberOfevaluations descr")
inputs = computation_job_inputs.ComputationJobInputs()
action1 = computational_job_input_action_update_json_file.ComputationalJobInputActionUpdateJSONFile(
"numberOfevaluations", target_json_file,
"['Config']['nEvaluations']", source_template_json_file)
inputs.add_input(param1)
inputs.set_values({"numberOfevaluations": "100"})
action1.set_inputs(inputs)
# check target_json_file
json_text = target_json_file.text()
j = json.loads(json_text)
self.assertTrue(100, j['Config']['nEvaluations'])
def test_instance_methods(self):
rc = Resource()
self.assertIsNone(rc.start_time())
self.assertIsNone(rc.cmd_line())
self.assertIsNone(rc.unique_id())
self.assertIsNone(rc.set_unique_id(1))
self.assertIsNone(rc.sendbox_path())
self.assertIsNone(rc.processbot_pid())
self.assertIsNone(rc.set_processbot_pid(1))
self.assertIsNone(rc.spawned_process_pids())
self.assertIsNone(rc.spawned_process_cmd())
sandbox_root = path("./src/wpsremote/xmpp_data/test/test_dir")
unique_id = str(random.randint(1, 1000)).zfill(5)
sendbox_path = sandbox_root / str(unique_id)
rc.set_from_servicebot(unique_id, sendbox_path)
self.assertEqual(rc.sendbox_path(), sendbox_path)
self.assertEqual(rc._unique_id, unique_id)
rc.set_from_processbot(os.getpid(), [unique_id])
self.assertIn(unique_id, rc._spawned_process_pids)
try:
rc.read()
except Exception as e:
self.fail(e)
def test_create_json_file_action(self):
p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", "par descr 1")
inputs = computation_job_inputs.ComputationJobInputs()
inputs.add_input( p1 )
actions = computational_job_input_actions.ComputationalJobInputActions()
a1 = computational_job_input_action_create_json_file.ComputationalJobInputActionCreateJSONFile("mypar1", path.path("./json_out_${json_path_expr}.json"), "['Asset']['id']", path.path("./src/wpsremote/xmpp_data/test/asset_schema.json"))
actions.add_actions( a1 )
inputs.set_values( { "mypar1" : TestComputationJobInputs.json_text1 } )
actions.execute( inputs )
self.assertTrue( a1.exists() )
def test_path_methods(self):
p = path.path("test/path")
self.assertEqual(p.__repr__(), "path('test/path')")
self.assertEqual(p.__add__("/add"), "test/path/add")
self.assertEqual(p.__radd__("radd/"), "radd/test/path")
self.assertEqual(p.__div__("rel"), "test/path/rel")
self.assertEqual(p.__truediv__("rel"), "test/path/rel")
self.assertFalse(p.isabs())
self.assertEqual(p.basename(), "path")
self.assertEqual(p.name, "path")
self.assertEqual(p.normcase(), "test/path")
self.assertEqual(p.normpath(), "test/path")
self.assertEqual(p.expanduser(), "test/path")
self.assertEqual(p.expandvars(), "test/path")
self.assertEqual(p.dirname(), "test")
self.assertEqual(p.parent, "test")
self.assertEqual(p.expand(), "test/path")
self.assertEqual(p._get_namebase(), "path")
self.assertEqual(p.namebase, "path")
self.assertEqual(p._get_ext(), "")
self.assertEqual(p.ext, "")
self.assertEqual(p._get_drive(), "")
self.assertEqual(p.drive, "")
self.assertEqual(p.splitpath(), (path.path('test'), 'path'))
self.assertEqual(p.splitdrive(),
(path.path(''), path.path('test/path')))
self.assertEqual(p.splitext(), (path.path('test/path'), ''))
self.assertEqual(p.stripext(), "test/path")
self.assertEqual(p.joinpath("join", "path"), "test/path/join/path")
self.assertEqual(p.splitall(), [path.path(''), 'test', 'path'])
self.assertEqual(p.relpath(), "test/path")
self.assertEqual(p.relpathto("dest"), "../../dest")
with self.assertRaises(OSError):
p.listdir(pattern=None)
existing_path = path.path("./src/wpsremote/xmpp_data/test")
self.assertEqual(len(existing_path.listdir(pattern=None)), 13)
for d in existing_path.listdir(pattern=None):
self.assertIn(d, TestPath.LIST_DIRS)
self.assertEqual(len(existing_path.dirs(pattern=None)), 2)
for d in existing_path.dirs(pattern=None):
self.assertIn(d, TestPath.DIRS)
self.assertEqual(len(existing_path.files()), 11)
for f in existing_path.files():
self.assertIn(f, TestPath.LIST_DIRS)
self.assertTrue(existing_path.fnmatch("*test*"))
self.assertEqual(
existing_path.glob("*test_dir*"),
[path.path('./src/wpsremote/xmpp_data/test/test_dir')])
existing_file = path.path("./src/wpsremote/xmpp_data/test/test_file")
if sys.version_info[0] < 3:
self.assertIsInstance(existing_file.open(), file)
else:
from io import IOBase
self.assertIsInstance(existing_file.open(), IOBase)
self.assertEqual(existing_file.bytes(), "test content")
existing_file.write_bytes("test file to write")
self.assertEqual(existing_file.bytes(), "test file to write")
self.assertEqual(existing_file.text(), "test file to write")
existing_file.write_text("test content")
self.assertEqual(existing_file.text(), "test content")
existing_file.write_lines(["line 1", "line 2"])
self.assertEqual(existing_file.lines(), ["line 1\n", "line 2\n"])
existing_file.write_text("test content")
# Methods for querying the filesystem
self.assertTrue(existing_file.exists())
self.assertFalse(path.path("./not/existing/path/test_file").exists())
self.assertTrue(existing_path.isdir())
self.assertTrue(existing_file.isfile())
self.assertTrue(path.path("/").ismount())
# mkdir
new_dir = path.path('./src/wpsremote/xmpp_data/test/new_dir')
new_dir.mkdir()
self.assertTrue(new_dir.exists())
new_dirs = path.path('./src/wpsremote/xmpp_data/test/new_dirs/new_dir')
new_dirs.makedirs()
self.assertTrue(new_dirs.exists())
# rmdir
new_dir.rmdir()
self.assertFalse(new_dir.exists())
new_dirs.removedirs()
self.assertFalse(new_dirs.exists())
# touch
touch_file = path.path('./src/wpsremote/xmpp_data/test/new_file')
self.assertFalse(touch_file.exists())
touch_file.touch()
self.assertTrue(touch_file.exists())
# remove
touch_file.remove()
self.assertFalse(touch_file.exists())
# unlink
touch_file.touch()
self.assertTrue(touch_file.exists())
touch_file.unlink()
self.assertFalse(touch_file.exists())
# copyfile
existing_file.copy('./src/wpsremote/xmpp_data/test/copy_of_test_file')
copied_file = path.path(
'./src/wpsremote/xmpp_data/test/copy_of_test_file')
self.assertTrue(copied_file.exists())
copied_file.remove()
def test_process_bot(self):
wps_agent = MockWPSAgentProcess(self.args)
# test bot
process_bot = wps_agent.create_bot()
self.assertIsInstance(process_bot, ProcessBot)
self.assertTrue(process_bot._active)
self.assertFalse(process_bot._finished)
self.assertIsInstance(process_bot._input_parameters_defs, ComputationJobInputs)
self.assertIsInstance(process_bot._input_params_actions, ComputationalJobInputActions)
self.assertEqual(process_bot._input_values, {"test_message": "test message"})
self.assertEqual(process_bot._max_running_time, datetime.timedelta(0, 10))
self.assertEqual(process_bot.max_execution_time(), datetime.timedelta(0, 10))
self.assertIsInstance(process_bot._output_parameters_defs, OutputParameters)
self.assertEqual(process_bot._remote_wps_baseurl, "test_base_url")
self.assertEqual(
process_bot._resource_file_dir,
path("./src/wpsremote/xmpp_data/test/resource_dir")
)
self.assertEqual(
process_bot.get_resource_file_dir(),
path("./src/wpsremote/xmpp_data/test/resource_dir")
)
self.assertEqual(len(process_bot._stdout_action), 6)
for stdout_action in process_bot._stdout_action:
self.assertIn(stdout_action, ['ignore', 'progress', 'log', 'log', 'log', 'abort'])
self.assertEqual(len(process_bot._stdout_parser), 6)
for stdout_parser in process_bot._stdout_parser:
self.assertIn(stdout_parser, [
'.*\\[DEBUG\\](.*)',
'.*\\[INFO\\] ProgressInfo\\:([-+]?[0-9]*\\.?[0-9]*)\\%',
'.*\\[(INFO)\\](.*)',
'.*\\[(WARN)\\](.*)',
'.*\\[(ERROR)\\](.*)',
'.*\\[(CRITICAL)\\](.*)'
])
self.assertEqual(process_bot._uniqueExeId, "123")
self.assertEqual(process_bot._uploader, None)
self.assertEqual(process_bot._wps_execution_shared_dir, None)
self.assertEqual(process_bot.get_wps_execution_shared_dir(), None)
self.assertEqual(process_bot.description, "foo service")
self.assertEqual(process_bot.namespace, "default")
self.assertEqual(process_bot.service, "Service")
self.assertEqual(
process_bot.workdir(),
process_bot._output_dir + "/" + process_bot._uniqueExeId
)
# handle_finish
finish_msg = {
"type": "normal",
"body": "topic=finish",
"from": self.from_obj_mock
}
finished_message = self.xmpp_bus.CreateIndipendentMessage(finish_msg)
with self.assertRaises(SystemExit) as cm_finish:
process_bot.handle_finish(finished_message)
self.assertEqual(cm_finish.exception.code, 0)
# handle_abort
abort_msg = {
"type": "normal",
"body": "topic=abort",
"from": self.from_obj_mock
}
abort_message = self.xmpp_bus.CreateIndipendentMessage(abort_msg)
with self.assertRaises(SystemExit) as cm_abort:
process_bot.handle_abort(abort_message)
self.assertEqual(cm_abort.exception.code, -1)
with self.assertRaises(SystemExit):
process_bot.exit(0)
# kill process
self.cleaner.kill_processbot()
self.assertFalse(psutil.pid_exists(self.cleaner._processbot_pid))
def test_service_bot(self):
wps_agent = MockWPSAgentService(self.args)
# test bot
service_bot = wps_agent.create_bot()
self.assertIsInstance(service_bot, ServiceBot)
self.assertTrue(service_bot._active)
self.assertIsInstance(service_bot._input_parameters_defs, ComputationJobInputs)
self.assertEqual(service_bot._max_running_time, datetime.timedelta(0, 10))
self.assertEqual(
service_bot._output_dir,
path('src\\wpsremote\\xmpp_data\\test\\tmp')
)
self.assertIsInstance(service_bot._output_parameters_defs, OutputParameters)
self.assertTrue(service_bot._redirect_process_stdout_to_logger)
self.assertEqual(
service_bot._remote_config_filepath,
'./src/wpsremote/xmpp_data/test/test_remote.config'
)
self.assertIsNone(service_bot._remote_wps_endpoint)
self.assertEqual(
service_bot._resource_file_dir,
path('./src/wpsremote/xmpp_data/test/resource_dir')
)
self.assertEqual(
service_bot.get_resource_file_dir(),
path("./src/wpsremote/xmpp_data/test/resource_dir")
)
self.assertIsInstance(service_bot._resource_monitor, MockResourceMonitor)
self.assertEqual(
service_bot._service_config_file,
'./src/wpsremote/xmpp_data/test/test_service.config'
)
self.assertIsNone(service_bot._wps_execution_shared_dir)
self.assertEqual(
service_bot.max_execution_time(),
datetime.timedelta(0, 10)
)
self.assertEqual(
service_bot.get_wps_execution_shared_dir(),
None
)
self.assertIsNotNone(service_bot.bus)
self.assertEqual(service_bot.description, 'foo service')
self.assertEqual(service_bot.namespace, 'default')
self.assertEqual(service_bot.running_process, {})
self.assertEqual(service_bot.service, 'Service')
self.assertEqual(
service_bot._process_blacklist,
json.loads(service_bot._remote_config.get("DEFAULT", "process_blacklist"))
)
self.assertEqual(
resource_monitor.ResourceMonitor.capacity,
service_bot._remote_config.getint("DEFAULT", "capacity")
)
self.assertEqual(
resource_monitor.ResourceMonitor.load_threshold,
service_bot._remote_config.getint("DEFAULT", "load_threshold")
)
self.assertEqual(
resource_monitor.ResourceMonitor.load_average_scan_minutes,
service_bot._remote_config.getint("DEFAULT", "load_average_scan_minutes")
)
PCk = None
try:
process_weight_class_name = service_bot._service_config.get("DEFAULT", "process_weight")
PCk = introspection.get_class_no_arg(process_weight_class_name)
except BaseException:
PCk = None
if not PCk:
try:
process_weight = json.loads(
service_bot._service_config.get("DEFAULT", "process_weight"))
except BaseException:
process_weight = {"weight": "0", "coefficient": "1.0"}
PCk = resource_monitor.ProcessWeight(process_weight)
self.assertEqual(PCk.__class__, resource_monitor.ProcessWeight)
_request_load = PCk.request_weight(None)
self.assertEqual(_request_load, 15.0)
# removing tmp files
params_file_path = path(self.params_file)
params_file_path.remove()
