def submit_experiment(
api,
name,
duration, # pylint:disable=too-many-arguments
resources,
start_time=None,
print_json=False):
""" Submit user experiment with JSON Encoder serialization object
Experiment and firmware(s). If submission is accepted by scheduler OAR
we print JSONObject response with id submission.
:param api: API Rest api object
:param name: experiment name
:param duration: experiment duration in seconds
:param resources: list of 'exp_resources' which
:param print_json: select if experiment should be printed as json instead
of submitted
"""
assert resources, 'Empty resources: %r' % resources
experiment = _Experiment(name, duration, start_time)
exp_files = helpers.FilesDict()
for res_dict in resources:
experiment.add_exp_resources(res_dict)
exp_files.add_firmware(res_dict.get('firmware', None)) # firmware
if print_json: # output experiment description
return experiment
# submit experiment
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment) # exp description
return api.submit_experiment(exp_files)
def test_experiment_script_run(self):
"""Test running experiment script run."""
experiment.script_experiment(
self.api, 123, 'run',
experiment.site_association(
'grenoble', script=tests.resource_file('script.sh'),
scriptconfig=tests.resource_file('scriptconfig')),
experiment.site_association(
'strasbourg', script=tests.resource_file('script_2.sh')),
)
scripts_json = helpers.json_dumps({
'script': [
{'scriptname': 'script.sh', 'sites': ['grenoble']},
{'scriptname': 'script_2.sh', 'sites': ['strasbourg']},
],
'scriptconfig': [
{'scriptconfigname': 'scriptconfig', 'sites': ['grenoble']},
],
})
expected_files = {
'script.json': scripts_json,
'script.sh': SCRIPTS['script.sh'],
'script_2.sh': SCRIPTS['script_2.sh'],
'scriptconfig': SCRIPTCONFIG['scriptconfig'],
}
self.api.script_command.assert_called_with(123, 'run',
files=expected_files)
def _script_run_files_dict(*site_associations):
"""Return script start files dict.
Returns dict with format
{
<RUN_FILENAME>: json({'script': [<scripts_associations>], ...}),
'scriptname': b'scriptcontent',
}
"""
if not site_associations:
raise ValueError('Got empty site_associations %r', site_associations)
_check_sites_uniq(*site_associations)
files_dict = helpers.FilesDict()
# Save association and files
associations = {}
for sites, assocs in site_associations:
for assoctype, assocname in assocs.items():
_add_siteassoc_to_dict(associations, sites, assoctype, assocname)
files_dict.add_files_from_dict(SITE_ASSOCIATIONS_FILE_ASSOCS, assocs)
# Add scrit sites association to files_dict
files_dict[RUN_FILENAME] = helpers.json_dumps(associations)
return files_dict
def _script_run_files_dict(*site_associations):
"""Return script start files dict.
Returns dict with format
{
<RUN_FILENAME>: json({'script': [<scripts_associations>], ...}),
'scriptname': b'scriptcontent',
}
"""
if not site_associations:
raise ValueError('Got empty site_associations: {}'
.format(site_associations))
_check_sites_uniq(*site_associations)
files_dict = helpers.FilesDict()
# Save association and files
associations = {}
for sites, assocs in site_associations:
for assoctype, assocname in assocs.items():
_add_siteassoc_to_dict(associations, sites, assoctype, assocname)
files_dict.add_files_from_dict(SITE_ASSOCIATIONS_FILE_ASSOCS, assocs)
# Add scrit sites association to files_dict
files_dict[RUN_FILENAME] = helpers.json_dumps(associations)
return files_dict
def submit_experiment(api, name, duration, # pylint:disable=too-many-arguments
resources, start_time=None, print_json=False):
""" Submit user experiment with JSON Encoder serialization object
Experiment and firmware(s). If submission is accepted by scheduler OAR
we print JSONObject response with id submission.
:param api: API Rest api object
:param name: experiment name
:param duration: experiment duration in seconds
:param resources: list of 'exp_resources' which
:param print_json: select if experiment should be printed as json instead
of submitted
"""
assert resources, 'Empty resources: %r' % resources
experiment = _Experiment(name, duration, start_time)
exp_files = helpers.FilesDict()
for res_dict in resources:
experiment.add_exp_resources(res_dict)
exp_files.add_firmware(res_dict.get('firmware', None)) # firmware
if print_json: # output experiment description
return experiment
# submit experiment
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment) # exp description
return api.submit_experiment(exp_files)
def test_method(self):
""" Test Api.method rest submission """
ret = {"test": "val"}
ret_val = RequestRet(200, content=json_dumps(ret))
m_req = patch("requests.request", return_value=ret_val).start()
# pylint:disable=protected-access
_auth = self.api.auth
# call get
ret = self.api.method("page")
m_req.assert_called_with("get", self._url + "page", files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
ret = self.api.method("page?1", "get")
m_req.assert_called_with("get", self._url + "page?1", files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
# call delete
ret = self.api.method("deeel", "delete")
m_req.assert_called_with("delete", self._url + "deeel", files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
# call post
ret = self.api.method("post_page", "post", json={})
m_req.assert_called_with("post", self._url + "post_page", files=None, json={}, auth=_auth)
self.assertEqual(ret, ret)
# call multipart
_files = {"entry": "{}"}
ret = self.api.method("multip", "post", files=_files)
m_req.assert_called_with("post", self._url + "multip", files=_files, json=None, auth=_auth)
self.assertEqual(ret, ret)
patch.stopall()
def test_experiment_script_run(self):
"""Test running experiment script run."""
experiment.script_experiment(
self.api, 123, 'run',
experiment.site_association(
'grenoble', script=tests.resource_file('script.sh'),
scriptconfig=tests.resource_file('scriptconfig')),
experiment.site_association(
'strasbourg', script=tests.resource_file('script_2.sh')),
)
scripts_json = helpers.json_dumps({
'script': [
{'scriptname': 'script.sh', 'sites': ['grenoble']},
{'scriptname': 'script_2.sh', 'sites': ['strasbourg']},
],
'scriptconfig': [
{'scriptconfigname': 'scriptconfig', 'sites': ['grenoble']},
],
})
expected_files = {
'script.json': scripts_json,
'script.sh': SCRIPTS['script.sh'],
'script_2.sh': SCRIPTS['script_2.sh'],
'scriptconfig': SCRIPTCONFIG['scriptconfig'],
}
self.api.script_command.assert_called_with(123, 'run',
files=expected_files)
def _assert_json_equal(self, assocs, expected):
"""Assert assocs give the expected result when json dumps and load.
Also do a 'manual' convert to dict/list.
"""
self.assertEqual(json.loads(helpers.json_dumps(assocs)), expected)
self.assertEqual([assoc.__dict__ for assoc in assocs], expected)
self.assertEqual(assocs.list(), expected)
def api_mock(ret=None):
""" Return a mock of an api object
returned value for api methods will be 'ret' parameter or API_RET
"""
ret = ret or API_RET
ret_val = RequestRet(content=json_dumps(ret), status_code=200) # HTTP OK
patch('requests.request', return_value=ret_val).start()
api_class = patch('iotlabcli.rest.Api').start()
api_class.return_value = Mock(wraps=Api('user', 'password'))
return api_class.return_value
def api_mock(ret=None):
""" Return a mock of an api object
returned value for api methods will be 'ret' parameter or API_RET
"""
ret = ret or API_RET
ret_val = RequestRet(content=json_dumps(ret), status_code=200) # HTTP OK
patch('requests.request', return_value=ret_val).start()
api_class = patch('iotlabcli.rest.Api').start()
api_class.return_value = Mock(wraps=Api('user', 'password'))
return api_class.return_value
def test_method(self):
""" Test Api.method rest submission """
ret = {'test': 'val'}
ret_val = RequestRet(200, content=json_dumps(ret))
m_req = patch('requests.request', return_value=ret_val).start()
# pylint:disable=protected-access
_auth = self.api.auth
# call get
ret = self.api.method('page')
m_req.assert_called_with('get',
self._url + 'page',
files=None,
json=None,
auth=_auth)
self.assertEqual(ret, ret)
ret = self.api.method('page?1', 'get')
m_req.assert_called_with('get',
self._url + 'page?1',
files=None,
json=None,
auth=_auth)
self.assertEqual(ret, ret)
# call delete
ret = self.api.method('deeel', 'delete')
m_req.assert_called_with('delete',
self._url + 'deeel',
files=None,
json=None,
auth=_auth)
self.assertEqual(ret, ret)
# call post
ret = self.api.method('post_page', 'post', json={})
m_req.assert_called_with('post',
self._url + 'post_page',
files=None,
json={},
auth=_auth)
self.assertEqual(ret, ret)
# call multipart
_files = {'entry': '{}'}
ret = self.api.method('multip', 'post', files=_files)
m_req.assert_called_with('post',
self._url + 'multip',
files=_files,
json=None,
auth=_auth)
self.assertEqual(ret, ret)
patch.stopall()
def main_cli(function, parser, args=None): # flake8: noqa
""" Main command-line execution. """
args = args or sys.argv[1:]
try:
parser_opts = parser.parse_args(args)
result = function(parser_opts)
print(helpers.json_dumps(result))
except (IOError, ValueError) as err:
parser.error(str(err))
except RuntimeError as err:
print("RuntimeError:\n{err!s}".format(err=err), file=sys.stderr)
sys.exit()
except KeyboardInterrupt: # pragma: no cover
print("\nStopped.", file=sys.stderr)
sys.exit()
def main_cli(function, parser, args=None): # flake8: noqa
""" Main command-line execution. """
args = args or sys.argv[1:]
try:
parser_opts = parser.parse_args(args)
result = function(parser_opts)
print(helpers.json_dumps(result))
except (IOError, ValueError) as err:
parser.error(str(err))
except RuntimeError as err:
print("RuntimeError:\n{err!s}".format(err=err), file=sys.stderr)
sys.exit()
except KeyboardInterrupt: # pragma: no cover
print("\nStopped.", file=sys.stderr)
sys.exit()
def _request(url, method='GET', auth=None, data=None):
"""
Call http `method` on url with `auth` and `data`
:param url: url to request.
:param method: request method
:param auth: HTTPBasicAuth object
:param data: request data
"""
if method == 'POST':
headers = {'content-type': 'application/json'}
req = requests.post(url, auth=auth, headers=headers,
data=helpers.json_dumps(data).encode('utf-8'))
elif method == 'MULTIPART':
req = requests.post(url, auth=auth, files=data)
elif method == 'DELETE':
req = requests.delete(url, auth=auth)
else:
req = requests.get(url, auth=auth)
return (req.status_code, req.content)
def reserve_nodes(self, login, exp_name,
nodes_list, start_time, duration):
"""
Submit a physical experiment (nodes list) and reservation date.
"""
# pylint:disable=W0212,R0913,E1123
logger.warning("iotlashell reserve_nodes")
exp_file = helpers.FilesDict()
_experiment = experiment._Experiment(exp_name, duration, start_time)
_experiment.type = 'physical'
_experiment.nodes = nodes_list
exp_file['new_exp.json'] = helpers.json_dumps(_experiment)
try:
return self.api.method('admin/experiments?user=%s' % login,
'post',
files=exp_file)
except HTTPError as err:
logger.warning("iotlashell reserve_nodes error %s" % err.reason)
return {'error' : err.reason}
def submit_experiment(
api,
name,
duration, # pylint:disable=too-many-arguments
resources,
start_time=None,
print_json=False,
sites_assocs=None):
""" Submit user experiment with JSON Encoder serialization object
Experiment and firmware(s). If submission is accepted by scheduler OAR
we print JSONObject response with id submission.
:param api: API Rest api object
:param name: experiment name
:param duration: experiment duration in minutes
:param resources: list of 'exp_resources'
:param print_json: select if experiment should be printed as json instead
of submitted
:param sites_assocs: list of 'site_association'
"""
assert resources, 'Empty resources: %r' % resources
experiment = _Experiment(name, duration, start_time)
exp_files = helpers.FilesDict()
for res_dict in resources:
experiment.add_exp_resources(res_dict)
exp_files.add_files_from_dict(NODES_ASSOCIATIONS_FILE_ASSOCS, res_dict)
sites_assocs = sites_assocs or ()
for site_assoc in sites_assocs:
experiment.add_site_association(site_assoc)
assocs = site_assoc.associations
exp_files.add_files_from_dict(SITE_ASSOCIATIONS_FILE_ASSOCS, assocs)
if print_json: # output experiment description
return experiment
# submit experiment
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment) # exp description
return api.submit_experiment(exp_files)
def _request(url, method='GET', auth=None, data=None):
"""
Call http `method` on url with `auth` and `data`
:param url: url to request.
:param method: request method
:param auth: HTTPBasicAuth object
:param data: request data
"""
if method == 'POST':
headers = {'content-type': 'application/json'}
req = requests.post(url,
auth=auth,
headers=headers,
data=helpers.json_dumps(data).encode('utf-8'))
elif method == 'MULTIPART':
req = requests.post(url, auth=auth, files=data)
elif method == 'DELETE':
req = requests.delete(url, auth=auth)
else:
req = requests.get(url, auth=auth)
return (req.status_code, req.content)
def test__method(self):
""" Test Api._method rest submission """
ret = {'test': 'val'}
ret_val = RequestRet(content=json_dumps(ret).encode('utf-8'),
status_code=200)
post = patch('requests.post', return_value=ret_val).start()
delete = patch('requests.delete', return_value=ret_val).start()
get = patch('requests.get', return_value=ret_val).start()
# pylint:disable=protected-access
_auth = Mock()
# call get
ret = rest.Api._method(self._url)
get.assert_called_with(self._url, auth=None)
self.assertEquals(ret, ret)
ret = rest.Api._method(self._url, method='GET', auth=_auth)
get.assert_called_with(self._url, auth=_auth)
self.assertEquals(ret, ret)
# call delete
ret = rest.Api._method(self._url, method='DELETE')
delete.assert_called_with(self._url, auth=None)
self.assertEquals(ret, ret)
# call post
ret = rest.Api._method(self._url, method='POST', data={})
post.assert_called_with(
self._url, data='{}'.encode('utf-8'),
headers={'content-type': 'application/json'},
auth=None)
self.assertEquals(ret, ret)
# call multipart
_files = {'entry': '{}'}
ret = rest.Api._method(self._url, method='MULTIPART', data=_files)
post.assert_called_with(self._url, files=_files, auth=None)
self.assertEquals(ret, ret)
patch.stopall()
def test_method(self):
""" Test Api.method rest submission """
ret = {'test': 'val'}
ret_val = RequestRet(200, content=json_dumps(ret))
m_req = patch('requests.request', return_value=ret_val).start()
# pylint:disable=protected-access
_auth = self.api.auth
# call get
ret = self.api.method('page')
m_req.assert_called_with('get', self._url + 'page',
files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
ret = self.api.method('page?1', 'get')
m_req.assert_called_with('get', self._url + 'page?1',
files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
# call delete
ret = self.api.method('deeel', 'delete')
m_req.assert_called_with('delete', self._url + 'deeel',
files=None, json=None, auth=_auth)
self.assertEqual(ret, ret)
# call post
ret = self.api.method('post_page', 'post', json={})
m_req.assert_called_with('post', self._url + 'post_page',
files=None, json={}, auth=_auth)
self.assertEqual(ret, ret)
# call multipart
_files = {'entry': '{}'}
ret = self.api.method('multip', 'post', files=_files)
m_req.assert_called_with('post', self._url + 'multip',
files=_files, json=None, auth=_auth)
self.assertEqual(ret, ret)
patch.stopall()
def load_experiment(api, exp_desc_path, firmware_list=()):
""" Load and submit user experiment description with firmware(s)
Firmwares required for experiment will be loaded from current directory,
except if their path is given in firmware_list
:param api: API Rest api object
:param exp_desc_path: path to experiment json description file
:param firmware_list: list of firmware path
"""
# 1. load experiment description
exp_dict = json.loads(helpers.read_file(exp_desc_path))
exp_files = helpers.FilesDict()
# 2. Add experiment description
exp_files[EXP_FILENAME] = helpers.json_dumps(exp_dict)
# 3. Add firmwares files to the experiment files using
# firmware_list and experiment firmwareassociations
# extract firmwares names
_fw_association = exp_dict['firmwareassociations'] or []
firmwares = set([assoc['firmwarename'] for assoc in _fw_association])
try:
# replace firwmare name by firmware_path from firmware_list
for _fw_path in firmware_list:
firmwares.remove(basename(_fw_path))
firmwares.add(_fw_path)
except KeyError as err:
raise ValueError("Firmware {!s} is not in experiment: {}".format(
err, exp_desc_path))
else:
# Add all firmwares to the experiment files
for _fw_path in firmwares:
exp_files.add_firmware(_fw_path)
return api.submit_experiment(exp_files)
def submit_experiment(api, name, duration, # pylint:disable=too-many-arguments
resources, start_time=None, print_json=False,
sites_assocs=None):
""" Submit user experiment with JSON Encoder serialization object
Experiment and firmware(s). If submission is accepted by scheduler OAR
we print JSONObject response with id submission.
:param api: API Rest api object
:param name: experiment name
:param duration: experiment duration in minutes
:param resources: list of 'exp_resources'
:param print_json: select if experiment should be printed as json instead
of submitted
:param sites_assocs: list of 'site_association'
"""
assert resources, 'Empty resources: %r' % resources
experiment = _Experiment(name, duration, start_time)
exp_files = helpers.FilesDict()
for res_dict in resources:
experiment.add_exp_resources(res_dict)
exp_files.add_files_from_dict(NODES_ASSOCIATIONS_FILE_ASSOCS, res_dict)
sites_assocs = sites_assocs or ()
for site_assoc in sites_assocs:
experiment.add_site_association(site_assoc)
assocs = site_assoc.associations
exp_files.add_files_from_dict(SITE_ASSOCIATIONS_FILE_ASSOCS, assocs)
if print_json: # output experiment description
return experiment
# submit experiment
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment) # exp description
return api.submit_experiment(exp_files)
def load_experiment(api, exp_desc_path, firmware_list=()):
""" Load and submit user experiment description with firmware(s)
Firmwares required for experiment will be loaded from current directory,
except if their path is given in firmware_list
:param api: API Rest api object
:param exp_desc_path: path to experiment json description file
:param firmware_list: list of firmware path
"""
# 1. load experiment description
exp_dict = json.loads(helpers.read_file(exp_desc_path))
exp_files = helpers.FilesDict()
# 2. Add experiment description
exp_files[EXP_FILENAME] = helpers.json_dumps(exp_dict)
# 3. Add firmwares files to the experiment files using
# firmware_list and experiment firmwareassociations
# extract firmwares names
_fw_association = exp_dict['firmwareassociations'] or []
firmwares = set([assoc['firmwarename'] for assoc in _fw_association])
try:
# replace firwmare name by firmware_path from firmware_list
for _fw_path in firmware_list:
firmwares.remove(basename(_fw_path))
firmwares.add(_fw_path)
except KeyError as err:
raise ValueError("Firmware {!s} is not in experiment: {}".format(
err, exp_desc_path))
else:
# Add all firmwares to the experiment files
for _fw_path in firmwares:
exp_files.add_firmware(_fw_path)
return api.submit_experiment(exp_files)
async def submit_experiment(session):
""" Submit experiment scenario """
users = molotov.get_var('users')
if users.empty():
print("No users ...")
assert False
user = users.get()
config = molotov.get_var('config')['experiments']
exp = _Experiment(name=generate_exp_name(),
duration=config['duration'])
alias = AliasNodes(config['nb_nodes'], molotov.get_var('site'), 'm3:at86rf231', False)
exp.set_alias_nodes(alias)
form = FormData()
form.add_field("exp", json_dumps(exp),
content_type="application/json")
async with session.post(
urljoin(molotov.get_var('url'), 'experiments'),
# password = Monkey-<login>
auth=get_auth(user, 'Monkey-{}'.format(user)),
data=form,
) as resp:
res = await resp.json()
assert res['id'] is not None
assert resp.status == 200
def load_experiment(api, exp_desc_path, files_list=()):
""" Load and submit user experiment description with firmware(s)
Firmwares and scripts required for experiment will be loaded from
current directory, except if their path is given in files_list
:param api: API Rest api object
:param exp_desc_path: path to experiment json description file
:param files_list: list of files path
"""
# 1. load experiment description
exp_dict = json.loads(helpers.read_file(exp_desc_path))
experiment = _Experiment.from_dict(exp_dict)
# 2. List files and update path with provided path
files = _files_with_filespath(experiment.filenames(), files_list)
# Construct experiment files
exp_files = helpers.FilesDict()
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment)
for exp_file in files:
exp_files.add_file(exp_file)
return api.submit_experiment(exp_files)
def load_experiment(api, exp_desc_path, files_list=()):
""" Load and submit user experiment description with firmware(s)
Firmwares and scripts required for experiment will be loaded from
current directory, except if their path is given in files_list
:param api: API Rest api object
:param exp_desc_path: path to experiment json description file
:param files_list: list of files path
"""
# 1. load experiment description
exp_dict = json.loads(helpers.read_file(exp_desc_path))
experiment = _Experiment.from_dict(exp_dict)
# 2. List files and update path with provided path
files = _files_with_filespath(experiment.filenames(), files_list)
# Construct experiment files
exp_files = helpers.FilesDict()
exp_files[EXP_FILENAME] = helpers.json_dumps(experiment)
for exp_file in files:
exp_files.add_file(exp_file)
return api.submit_experiment(exp_files)