def _compose_arg(self, variables):
"""
Search for the method in generated _pb2. Resolve input type by the method's input.
"""
schema = fill_template(self.schema, variables)
service = fill_template(self.service, variables)
method = fill_template(self.method, variables)
# search for the service
mod = module_utils.load_external_actions(
join(variables['RESOURCES_DIR'],
file_utils.get_filename(schema) + '_pb2.py'))
services = {
str(k).lower(): v
for k, v in mod.DESCRIPTOR.services_by_name.items()
}
service = services.get(service)
if not service:
raise Exception('Unable to find service {} in {}'.format(
service,
file_utils.get_filename(schema) + '_pb2'))
# find service's method
methods = dict([(f.name.lower(), f) for f in service.methods])
method = methods.get(method)
if not method:
raise Exception('No method {} in service {}'.format(
method, service))
# find method's input type
input_type = method.input_type
classes = module_utils.get_all_classes(mod)
data = try_get_objects(fill_template_str(self.data, variables))
return method.name, classes[input_type.name](**data)
def action(self, includes: dict, variables: dict) -> dict or tuple:
cmd = fill_template(self._cmd, variables)
return_code, stdout, stderr = external_utils.run_cmd(cmd.split(' '),
variables,
fill_template(self._path, variables))
if return_code != int(fill_template(self._return_code, variables)):
debug('Process return code {}.\nStderr is {}\nStdout is {}'.format(return_code, stderr, stdout))
raise Exception(stderr)
return variables, stdout
def action(self, includes: dict, variables: dict) -> tuple:
out = fill_template(self.source, variables)
if self.dst is None:
info(out)
else:
dst = fill_template(self.dst, variables)
with open(join(self.path, dst), 'w') as f:
f.write(str(out))
return variables, out
def operation(self, variables: dict):
body = self.subject[self.body]
source = fill_template(self.determine_source(body), variables)
subject = fill_template(body['the'], variables)
result = subject in source
if self.negative:
result = not result
if not result:
debug(str(subject) + ' is not in ' + str(source))
return result
def test_template_with_complex_object(self):
my_dict = {'key': 'value', 'inner': {'key': 'value'}}
self.assertEqual(my_dict,
fill_template('{{ OUTPUT }}', {'OUTPUT': my_dict}))
my_list = [1, 2, 3, [1, 2, 3]]
self.assertEqual(my_list,
fill_template('{{ OUTPUT }}', {'OUTPUT': my_list}))
my_tuple = (1, 2, (1, 2))
self.assertEqual(my_tuple,
fill_template('{{ OUTPUT }}', {'OUTPUT': my_tuple}))
my_complex = [(1, {'foo': [1, 2, 3]}, [1, 2, {1: 'a'}])]
self.assertEqual(my_complex,
fill_template('{{ OUTPUT }}', {'OUTPUT': my_complex}))
def process_register(self,
variables,
output: dict or list or str or None = None) -> dict:
if self.register is not None:
for key in self.register.keys():
if output is not None:
out = fill_template(
self.register[key],
merge_two_dicts(variables,
{'OUTPUT': try_get_object(output)}))
else:
out = fill_template(self.register[key], variables)
variables[key] = out
return variables
def action(self, includes: dict, variables: dict) -> dict or tuple:
cmd = fill_template(self._cmd, variables)
process = subprocess.Popen(cmd.split(' '),
cwd=fill_template(self._path, variables),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
stdout, stderr = process.communicate()
if process.returncode != int(
fill_template(self._return_code, variables)):
debug('Process return code {}.\nStderr is {}\nStdout is {}'.format(
process.returncode, stderr, stdout))
raise Exception(stderr)
return variables, stdout
def _open_channel(self, channel, variables):
"""
Search for stub in generated module _pb2_grpc. Instantiate it with channel and return.
"""
schema = fill_template(self.schema, variables)
mod = module_utils.load_external_actions(
join(variables['RESOURCES_DIR'],
file_utils.get_filename(schema) + '_pb2_grpc.py'))
classes = module_utils.get_all_classes(mod)
classes = {k.lower(): v for k, v in classes.items()}
stub = classes.get(fill_template(self.service, variables) + 'stub')
if not stub:
raise Exception(
'Can\'t find stub in generated code. Something went wrong')
return stub(channel)
def action(self, includes: dict, variables: dict) -> tuple:
if self.source_file: # read from file
resources = variables['RESOURCES_DIR']
out = fill_template_str(
read_file(os.path.join(resources, self.source_file)),
variables)
else:
out = fill_template(self.source, variables)
if self.dst is None:
info(out)
else:
dst = fill_template(self.dst, variables)
with open(join(self.path, dst), 'w') as f:
f.write(str(out))
return variables, out
def operation(self, variables: dict) -> bool:
if isinstance(self.subject, str):
subject = fill_template(self.subject, variables)
source = True
else:
body = self.subject[self.body]
if isinstance(body, str):
body = Equals.to_long_form(body, True)
subject = fill_template(body['the'], variables)
source = fill_template(self.determine_source(body), variables)
result = source == subject
if self.negative:
result = not result
if not result:
debug(str(source) + ' is not equal to ' + str(subject))
return result
def action(self, includes: dict, variables: dict) -> Union[tuple, dict]:
url = fill_template(self.url, variables)
session = Http.sessions.get(self.session, requests.Session())
r = None
try:
r = session.request(self.method, url,
**self._form_request(url, variables))
if self._should_fail: # fail expected
raise RuntimeError('Request expected to fail, but it doesn\'t')
except requests.exceptions.ConnectionError as e:
debug(str(e))
if self._should_fail: # fail expected
return variables
self.__fix_cookies(url, session)
if self.session is not None: # save session if name is specified
Http.sessions[self.session] = session
if r is None:
raise Exception('No response received')
debug(r.text)
try:
response = r.json()
except ValueError:
response = r.text
if self.__check_code(r.status_code, self.code):
raise RuntimeError('Code mismatch: ' + str(r.status_code) +
' vs ' + str(self.code))
return variables, response
def operation(self, variables) -> bool:
body = self.subject[self.body]
source = fill_template(body['of'], variables)
if isinstance(source, list):
elements = source
elif isinstance(source, dict):
elements = source.items()
else:
debug(str(source) + ' not iterable')
return False
results = []
for element in elements:
oper_body = dict([(k, v) for (k, v) in body.items() if k != 'of'])
[next_operator] = oper_body.keys()
if not isinstance(oper_body[next_operator],
dict): # terminator in short form
if next_operator == 'equals':
oper_body[next_operator] = Equals.to_long_form(
'{{ ITEM }}', oper_body[next_operator])
if next_operator == 'contains':
oper_body[next_operator] = Contains.to_long_form(
'{{ ITEM }}', oper_body[next_operator])
next_operation = Operator.find_operator(oper_body)
variables['ITEM'] = element
results.append(next_operation.operation(variables))
return self.operator(results)
def action(self, includes: dict, variables: dict) -> dict:
filled_vars = dict([(k, fill_template(v, variables))
for (k, v) in self.variables.items()])
out = fill_template_str(self.include, variables)
test, tag = get_tag(out)
if test not in includes:
error('No include registered for name ' + test)
raise Exception('No include registered for name ' + test)
include = includes[test]
variables = merge_two_dicts(include.variables,
merge_two_dicts(variables, filled_vars))
include.variables = try_get_object(
fill_template_str(variables, variables))
try:
info('Running {}.{}'.format(test, '' if tag is None else tag))
logger.log_storage.nested_test_in()
variables = include.run(tag=tag, raise_stop=True)
logger.log_storage.nested_test_out()
except SkipException:
logger.log_storage.nested_test_out()
debug('Include ignored')
return variables
except StopException as e:
logger.log_storage.nested_test_out()
raise e
except Exception as e:
logger.log_storage.nested_test_out()
if not self.ignore_errors:
raise Exception('Step run ' + test + ' failed: ' + str(e))
return variables
def action(self, includes: dict, variables: dict) -> tuple:
if self.source_file: # read from file
resources = variables['RESOURCES_DIR']
out = fill_template_str(read_file(join(resources, fill_template_str(self.source_file, variables))),
variables)
else:
out = fill_template(self.source, variables)
if self.dst is None:
info(out)
else:
dst = fill_template(self.dst, variables)
path = fill_template(self.path, variables)
filename = join(path, dst)
file_utils.ensure_dir(os.path.dirname(os.path.abspath(filename)))
with open(filename, 'w') as f:
f.write(str(out))
return variables, out
def _compile_proto_files(self, variables):
"""
Compile .proto resource into the definition (_pb2) and client code (_pb2_grpc)
"""
from grpc.tools import command
schema = fill_template(self.schema, variables)
command.build_package_protos(join(variables['RESOURCES_DIR'],
dirname(schema)),
strict_mode=True)
def test_render_date_nested(self):
now = datetime.datetime.now()
res = fill_template('{{ OUTPUT }}',
{'OUTPUT': {
'key': 'value1',
'date': now
}})
self.assertEqual(res['date'], now.strftime('%Y-%m-%d %H:%M:%S.%f'))
self.assertEqual(res['key'], 'value1')
def action(self, includes: dict, variables: dict) -> dict or tuple:
import grpc
channel = grpc.insecure_channel(fill_template(self.url, variables))
if self.schema:
self._compile_proto_files(variables)
client = self._open_channel(channel, variables)
method, input_arg = self._compose_arg(variables)
return variables, getattr(client, method)(input_arg)
else:
raise Exception('Reflection not supported (yet)')
def __form_body(self, variables) -> str or dict:
if self.method == 'get':
return False, None
body = self.body
if body is None:
body = read_file(fill_template_str(self.file, variables))
if isinstance(body, dict): # dump body to json to be able fill templates in
body = json.dumps(body)
isjson = 'tojson' in body
return isjson, fill_template(body, variables, isjson=isjson)
def test_template_with_simple_object(self):
self.assertEqual(True, fill_template('{{ OUTPUT }}', {'OUTPUT': True}))
self.assertEqual(17, fill_template('{{ OUTPUT }}', {'OUTPUT': 17}))
self.assertEqual(4.2, fill_template('{{ OUTPUT }}', {'OUTPUT': 4.2}))
self.assertEqual('string',
fill_template('{{ OUTPUT }}', {'OUTPUT': 'string'}))
self.assertEqual('id', fill_template('{{ OUTPUT }}', {'OUTPUT': 'id'}))
self.assertEqual(
'2020-03-11',
fill_template('{{ OUTPUT }}', {'OUTPUT': '2020-03-11'}))
def __form_body(self, variables) -> tuple:
if self.method == 'get':
return False, None
body = self.body
if body is None and self.file is not None:
resources = variables['RESOURCES_DIR']
body = file_utils.read_file(
fill_template_str(os.path.join(resources, self.file),
variables))
if isinstance(body, dict) or isinstance(
body, list): # dump body to json to be able fill templates in
body = json.dumps(body)
if body is None:
return False, None
isjson = 'tojson' in body
return isjson, fill_template(body, variables, isjson=isjson)
def action(self, includes: dict, variables: dict) -> Union[tuple, dict]:
url = fill_template(self.url, variables)
r = None
try:
r = request(self.method, url, **self._form_request(url, variables))
if self._should_fail: # fail expected
raise RuntimeError('Request expected to fail, but it doesn\'t')
except requests.exceptions.ConnectionError as e:
debug(str(e))
if self._should_fail: # fail expected
return variables
debug(r.text)
if r.status_code != self.code:
raise RuntimeError('Code mismatch: ' + str(r.status_code) + ' vs ' + str(self.code))
try:
response = r.json()
except ValueError:
response = r.text
return variables, response
def action(self, includes: dict, variables: dict) -> any:
# if virtual host is not specified default it to /
config = try_get_objects(fill_template_str(self.config, variables))
if config.get('virtualhost') is None:
config['virtualhost'] = ''
disconnect_timeout = int(config.get('disconnect_timeout', 10)) # 10 sec for connection closed exception
connection_parameters = self._get_connection_parameters(config)
if self.method == 'publish':
message = self.form_body(self.message, self.file, variables)
return variables, self.publish(connection_parameters,
fill_template_str(self.exchange, variables),
fill_template_str(self.routing_key, variables),
fill_template(self.headers, variables),
message,
disconnect_timeout)
elif self.method == 'consume':
return variables, self.consume(connection_parameters,
fill_template_str(self.queue, variables),
disconnect_timeout)
else:
raise AttributeError('unknown method: ' + self.method)
