async def request_locker(servers: t.Union[Server, t.List[Server]],
action,
scope,
applicant,
auth=None,
datemark=None) -> t.List[Response]:
tasks = []
server_responses = []
if is_iterable_not_string(servers):
it = servers
else:
it = [servers]
payload = dict(scope=scope.name, applicant=applicant)
if datemark:
payload.update(datemark=datemark)
for server in it:
tasks.append(
create_task(
async_post(server,
'api_1_0.locker_' + action,
json=payload,
auth=auth,
timeout=defaults.TIMEOUT_LOCK_REQUEST)))
for server in it:
r = await tasks.pop(0)
server_responses.append(r)
return server_responses
def lock_scope(scope: Scope,
servers: t.Union[t.List[Server], Server] = None,
bypass: t.Union[t.List[Server], Server] = None,
retries=0,
delay=3,
applicant=None,
identity=None):
if not locker_scope_enabled(scope):
yield None
return
if servers is not None:
servers = servers if is_iterable_not_string(servers) else [servers]
if len(servers) == 0:
servers = None
servers = servers or get_servers_from_scope(scope, bypass)
logger.debug(
f"Requesting Lock on {scope.name} to the following servers: {[s.name for s in servers]}"
)
_try = 1
applicant = lock(scope,
servers=servers,
applicant=applicant,
retries=retries,
delay=delay,
identity=identity)
try:
yield applicant
finally:
unlock(scope, servers=servers, applicant=applicant, identity=identity)
def _execute(self, params: Kwargs, timeout=None, context: Context = None):
input_params = params['input']
cp = CompletedProcess()
cp.set_start_time()
try:
with lock_scope(Scope.CATALOG,
retries=3,
delay=4,
applicant=context.env.get('orch_execution_id')):
server_names = input_params.get('server_names', [])
if not is_iterable_not_string(server_names):
server_names = [server_names]
to_be_deleted = Server.query.filter(
Server.name.in_(server_names)).all()
for s in to_be_deleted:
# remove associated routes
db.session.delete(s.route)
s.delete()
db.session.commit()
except errors.LockError as e:
cp.success = False
cp.stderr = str(e)
except Exception as e:
cp.success = False
cp.stderr = f"Unable to delete server{'s' if len(input_params.get('to_be_deleted', [])) > 1 else ''} " \
f"{', '.join([s.name for s in to_be_deleted])}. Exception: {e}"
else:
cp.success = True
cp.stdout = f"Server{'s' if len(to_be_deleted) > 1 else ''} " \
f"{', '.join([s._old_name for s in to_be_deleted])} deleted"
cp.set_end_time()
return cp
def patch(self, orchestration_id):
o = Orchestration.query.get_or_raise(orchestration_id)
for k, v in request.get_json().items():
if k == 'description':
v = '\n'.join(v) if is_iterable_not_string(v) else v
setattr(o, k, v)
db.session.commit()
return {}, 204
def __init__(self, entity: str, ident, columns: t.Iterable[str] = None):
self.entity = entity
self.ident = ident
if is_iterable_not_string(columns):
self.columns = columns
elif is_string_types(columns):
self.columns = [columns]
else:
self.columns = ['id']
def _execute(self, params: Kwargs, timeout=None, context: Context = None):
input_params = params['input']
start = time.time()
running = []
cp = CompletedProcess()
cp.set_start_time()
try:
min_timeout = min(timeout, input_params.get('timeout', None))
except TypeError:
if timeout is not None:
min_timeout = timeout
elif input_params.get('timeout', None) is not None:
min_timeout = input_params.get('timeout')
else:
min_timeout = defaults.MAX_TIME_WAITING_SERVERS
server_names = input_params.get('server_names', [])
if not is_iterable_not_string(server_names):
server_names = [server_names]
if not server_names:
cp.success = False
cp.stdout = f"No server to wait for DM running"
cp.set_end_time()
return cp
pending_names = set(server_names)
found_names = []
while len(pending_names) > 0:
try:
found_names = db.session.query(Server.name).join(
Route, Route.destination_id == Server.id).filter(
Server.name.in_(pending_names)).filter(
Route.cost.isnot(None)).order_by(
Server.name).all()
except sqlite3.OperationalError as e:
if str(e) == 'database is locked':
pass
else:
raise
found_names = set([t[0] for t in found_names])
pending_names = pending_names - found_names
if pending_names and (time.time() - start) < min_timeout:
time.sleep(self.system_kwargs.get('sleep_time', 15))
else:
break
if not pending_names:
cp.success = True
cp.stdout = f"Server{'s' if len(server_names) > 1 else ''} " \
f"{', '.join(sorted(server_names))} with dimensigon running"
else:
cp.success = False
cp.stderr = f"Server{'s' if len(pending_names) > 1 else ''} {', '.join(sorted(pending_names))} " \
f"with dimensigon not running after {min_timeout} seconds"
cp.set_end_time()
return cp
def get_completions(self, document: Document, complete_event: CompleteEvent, var_filters: dict = None) -> \
Iterable[Completion]:
url_filters = {}
for filter in self.filters:
source, dest = filter
if var_filters:
value = var_filters.get(dest, None)
if value:
if is_iterable_not_string(value):
value = ','.join(value)
url_filters.update({f'filter[{dest}]': value})
try:
url = ntwrk.generate_url(self.resource, {
**url_filters,
**self.resource_params
})
except:
return
res = ntwrk.request('get', url, login=False, timeout=3)
if res.code == 200:
words = []
meta_words = {}
for e in res.msg:
if isinstance(e, dict) and e.get(self.key) not in words:
words.append(e.get(self.key))
if self.meta_key or self.meta_format:
if self.meta_format:
if self.transforms:
for key, transform in self.transforms.items():
if key in e:
val = e.get(key)
try:
e['key'] = transform(val)
except:
pass
try:
format = self.meta_format.format(**e)
except:
pass
else:
meta_words[e.get(self.key)] = HTML(format)
else:
meta_words[e.get(self.key)] = HTML(
e.get(self.meta_key))
elif isinstance(e, str):
words.append(e)
completer = DshellWordCompleter(words,
ignore_case=self.ignore_case,
match_middle=self.match_middle,
meta_dict=meta_words)
for c in completer.get_completions(document, complete_event):
yield c
def __init__(self,
name: str,
version: int,
action_type: ActionType,
code: MultiLine = None,
expected_stdout: MultiLine = None,
expected_stderr: MultiLine = None,
expected_rc: int = None,
system_kwargs: typos.Kwargs = None,
pre_process: MultiLine = None,
post_process: MultiLine = None,
schema: typos.Kwargs = None,
description: MultiLine = None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self.version = version
self.action_type = action_type
self.code = '\n'.join(code) if is_iterable_not_string(code) else code
self.schema = schema or {}
self.expected_stdout = '\n'.join(
expected_stdout) if is_iterable_not_string(
expected_stdout) else expected_stdout
self.expected_stderr = '\n'.join(
expected_stderr) if is_iterable_not_string(
expected_stderr) else expected_stderr
self.expected_rc = expected_rc
self.system_kwargs = system_kwargs or {}
self.pre_process = '\n'.join(pre_process) if is_iterable_not_string(
pre_process) else pre_process
self.post_process = '\n'.join(post_process) if is_iterable_not_string(
post_process) else post_process
self.description = '\n'.join(description) if is_iterable_not_string(
description) else description
def _replace_path_args(path, args):
replaced_path = path
match = re.search(r'\<((\w+:)?([\w_]+))\>', path)
while match:
text = match.groups()[2]
assert text in args
value = args.pop(text)
if not value:
raise ValueError(f"No value specified for '{text}' in URL {path}")
replaced_path = "{}{}{}".format(replaced_path[:match.start()], value,
replaced_path[match.end():])
match = re.search(r'\<((\w+:)?([\w_]+))\>', replaced_path)
params = []
if args:
for k, v in args.items():
if is_iterable_not_string(v):
for vv in v:
if vv is not None:
params.append(f"{k}={urllib.parse.quote_plus(vv)}")
else:
if v is not None:
params.append(f"{k}={urllib.parse.quote_plus(v)}")
return replaced_path + '?' + '&'.join(params)
def launch_command():
data = request.get_json()
server_list = []
if 'target' in data:
not_found = []
servers = Server.query.all()
if data['target'] == 'all':
server_list = servers
elif is_iterable_not_string(data['target']):
for vv in data['target']:
sl = search(vv, servers)
if len(sl) == 0:
not_found.append(vv)
else:
server_list.extend(sl)
else:
sl = search(data['target'], servers)
if len(sl) == 0:
not_found.append(data['target'])
else:
server_list.extend(sl)
if not_found:
return {
'error':
"Following granules or ids did not match to any server: " +
', '.join(not_found)
}, 404
else:
server_list.append(g.server)
if re.search(r'rm\s+((-\w+|--[-=\w]*)\s+)*(-\w*[rR]\w*|--recursive)',
data['command']):
return {'error': 'rm with recursion is not allowed'}, 403
data.pop('target', None)
start = None
username = getattr(User.query.get(get_jwt_identity()), 'name', None)
if not username:
raise errors.EntityNotFound('User', get_jwt_identity())
cmd = wrap_sudo(username, data['command'])
if g.server in server_list:
start = time.time()
server_list.pop(server_list.index(g.server))
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE if data.get('input', None) else None,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True,
close_fds=True,
encoding='utf-8')
resp_data = {}
if check_param_in_uri("human"):
attr = 'name'
else:
attr = 'id'
if server_list:
resp: t.List[ntwrk.Response] = asyncio.run(
ntwrk.parallel_requests(server_list,
method='POST',
view_or_url='api_1_0.launch_command',
json=data))
for s, r in zip(server_list, resp):
key = getattr(s, attr, s.id)
if r.ok:
resp_data[key] = r.msg[s.id]
else:
if not r.exception:
resp_data[key] = {
'error': {
'status_code': r.code,
'response': r.msg
}
}
else:
if isinstance(r.exception, errors.BaseError):
resp_data[key] = errors.format_error_content(
r.exception, current_app.config['DEBUG'])
else:
resp_data[key] = {
'error':
format_exception(r.exception) if
current_app.config['DEBUG'] else str(r.exception)
or str(r.exception.__class__.__name__)
}
if start:
key = getattr(g.server, attr, g.server.id)
timeout = data.get('timeout', defaults.TIMEOUT_COMMAND)
try:
outs, errs = proc.communicate(input=(data.get('input', '') or ''),
timeout=timeout)
except (TimeoutError, subprocess.TimeoutExpired):
proc.kill()
try:
outs, errs = proc.communicate(timeout=1)
except:
resp_data[key] = {
'error':
f"Command '{cmd}' timed out after {timeout} seconds. Unable to communicate with the process launched."
}
else:
resp_data[key] = {
'error':
f"Command '{cmd}' timed out after {timeout} seconds",
'stdout': outs.split('\n'),
'stderr': errs.split('\n')
}
except Exception as e:
current_app.logger.exception(
"Exception raised while trying to run command")
resp_data[key] = {
'error':
traceback.format_exc() if current_app.config['DEBUG'] else
str(r.exception) or str(r.exception.__class__.__name__)
}
else:
resp_data[key] = {
'stdout': outs.split('\n'),
'stderr': errs.split('\n'),
'returncode': proc.returncode
}
resp_data['cmd'] = cmd
resp_data['input'] = data.get('input', None)
return resp_data, 200
def lock(scope: Scope,
servers: t.List[Server] = None,
applicant=None,
retries=0,
delay=3,
identity=None) -> UUID:
"""
locks the Locker if allowed
Parameters
----------
scope
scope that lock will affect.
servers
if scope set to Scope.ORCHESTRATION,
applicant
identifier of the lock
Returns
-------
Result
"""
if servers is not None:
servers = servers if is_iterable_not_string(servers) else [servers]
if len(servers) == 0:
servers = None
servers = servers or get_servers_from_scope(scope)
if len(servers) == 0:
raise errors.NoServerToLock(scope)
applicant = applicant if applicant is not None else [
str(s.id) for s in servers
]
_try = 1
while True:
try:
lock_unlock(action='L',
scope=scope,
servers=servers,
applicant=applicant,
identity=identity)
except errors.LockError as e:
if _try < retries:
_try += 1
logger.info(
f"Retrying to lock on {scope.name} in {delay} seconds")
time.sleep(delay)
else:
error_servers = [r.server for r in e.responses]
locked_servers = list(
set(server for server in servers) - set(error_servers))
lock_unlock('U',
scope,
servers=locked_servers,
applicant=applicant,
identity=identity)
raise e
else:
break
return applicant
async def _send_routes(self, exclude=None):
servers = Server.get_neighbours(session=self.session)
msg, debug_msg = self._format_routes_message(self._changed_routes)
c_exclude = []
if self.logger.level <= logging.DEBUG:
if exclude:
if is_iterable_not_string(exclude):
c_exclude = [
self.session.query(Server).get(e)
if not isinstance(e, Server) else e for e in exclude
]
else:
c_exclude = [
self.session.query(Server).get(exclude)
if not isinstance(exclude, Server) else exclude
]
log_msg = f" (Excluded nodes: {', '.join([getattr(e, 'name', e) for e in c_exclude])}):"
else:
log_msg = ''
if servers:
log_msg = f"Sending route information to the following nodes: {', '.join([s.name for s in servers])} " \
f"{log_msg}{json.dumps(debug_msg, indent=2)}"
else:
log_msg = f"No servers to send new routing information:{log_msg}{json.dumps(debug_msg, indent=2)}"
if debug_msg:
log_msg += '\n' + json.dumps(debug_msg, indent=2)
if debug_msg and (servers or exclude):
self.logger.debug(log_msg)
exclude_ids = list(
set([s.id for s in servers
]).union([getattr(e, 'id', e) for e in c_exclude]))
auth = get_root_auth()
aw = [
ntwrk.async_patch(s,
view_or_url='api_1_0.routes',
json={
'server_id': self.server.id,
'route_list': msg,
'exclude': exclude_ids
},
auth=auth) for s in servers
]
rs = await asyncio.gather(*aw, return_exceptions=True)
for r, s in zip(rs, servers):
if isinstance(r, Exception):
self.logger.warning(
f"Error while trying to send route data to node {s}: "
f"{format_exception(r)}")
elif not r.ok:
if r.exception:
self.logger.warning(
f"Error while trying to send route data to node {s}: "
f"{format_exception(r.exception)}")
else:
self.logger.warning(
f"Error while trying to send route data to node {s}: {r}"
)
self._changed_routes.clear()
def _execute(self, params: Kwargs, timeout=None, context: Context = None):
input_params = params['input']
start = time.time()
found = []
cp = CompletedProcess()
cp.set_start_time()
timeout = input_params.get('timeout')
if timeout is None:
timeout = self.system_kwargs.get('timeout')
if timeout is None:
timeout = defaults.MAX_TIME_WAITING_SERVERS
try:
now = get_now()
server_names = input_params.get('server_names', [])
if not is_iterable_not_string(server_names):
server_names = [server_names]
if not server_names:
cp.success = False
cp.stderr = f"No server to wait"
cp.set_end_time()
return cp
pending_names = set(server_names)
with lock_scope(Scope.CATALOG,
retries=3,
delay=4,
applicant=context.env.get('orch_execution_id')):
while len(pending_names) > 0:
try:
found_names = db.session.query(Server.name).filter(
Server.name.in_(pending_names)).filter(
Server.created_on >= now).all()
except sqlite3.OperationalError as e:
if str(e) == 'database is locked':
found_names = []
else:
raise
found_names = set([t[0] for t in found_names
]) if found_names else set()
pending_names = pending_names - found_names
if pending_names and (time.time() - start) < timeout:
time.sleep(self.system_kwargs.get('sleep_time', 15))
else:
break
except errors.LockError as e:
cp.success = False
cp.stderr = str(e)
else:
if not pending_names:
cp.success = True
cp.stdout = f"Server{'s' if len(server_names) > 1 else ''} " \
f"{', '.join(sorted(server_names))} found"
else:
cp.success = False
cp.stderr = f"Server{'s' if len(pending_names) > 1 else ''} {', '.join(sorted(pending_names))} " \
f"not created after {timeout} seconds"
cp.set_end_time()
return cp
def get_completions(
self, document: Document,
complete_event: CompleteEvent) -> t.Iterable[Completion]:
# TODO: Problem with completing positionals. Solve argument resolution to know in which positional.
try:
# Split document.
text = document.text_before_cursor.lstrip()
stripped_len = len(document.text_before_cursor) - len(text)
if text.endswith('-h') or text.endswith('--help'):
return
# If there is a space, check for the first term, and use a
# subcompleter.
if " " in text:
first_term = text.split()[0]
completer = self.options.get(first_term)
# If we have a sub completer, use this for the completions.
if isinstance(completer, Completer):
remaining_text = text[len(first_term):].lstrip()
move_cursor = len(text) - len(
remaining_text) + stripped_len
new_document = Document(
remaining_text,
cursor_position=document.cursor_position - move_cursor,
)
for c in completer.get_completions(new_document,
complete_event):
yield c
# we reached the bottom subcommand. Parse to see if we have to autocomplete an argument or its value
else:
options = {}
params = {}
dest_args = {}
if not completer:
return
for arg in completer:
if isinstance(arg, dict):
arg = [arg]
elif isinstance(arg, list):
pass
else:
# to pass command function in dict command definition
continue
for a in arg:
if a.get('argument').startswith('-'):
options.update({a.get('argument'): a})
else:
params.update({a.get('argument'): a})
if 'dest' in a:
dest_args.update({a.get('dest'): a})
else:
dest_args.update(
{a.get('argument').lstrip('-'): a})
try:
words = shlex.split(text)
except:
return
if len(words
) > 0 and words[-1] in options and text.endswith(
words[-1]):
completer = DshellWordCompleter(
words=list(options.keys()))
for c in completer.get_completions(
document, complete_event):
yield c
for p in params:
if 'choices' in params[p]:
completer = DshellWordCompleter(
params[p].get("choices"))
elif 'completer' in params[p]:
completer = params[p].get('completer')
else:
parser = create_parser(
completer, GuessArgumentParser(allow_abbrev=False))
finder = "F:I.N:D.E:R"
if document.char_before_cursor == ' ':
text = document.text + finder
current_word = finder
else:
text = document.text
current_word = document.get_word_before_cursor(
WORD=True)
namespace = parser.parse_args(shlex.split(text)[1:])
values = dict(namespace._get_kwargs())
# find key related to current_word
for k, v in values.items():
if is_iterable_not_string(v):
if current_word in v:
break
else:
if v == current_word:
break
else:
k = None
v = None
# special case for DictAction
for dest, arg_def in dest_args.items():
if 'action' in arg_def and arg_def[
'action'] == DictAction and values[dest]:
for k, v in values[dest].items():
# target
if k == current_word:
resp = ntwrk.get(
'api_1_0.orchestrationresource',
view_data={
'orchestration_id':
values['orchestration_id']
})
if resp.ok:
needed_target = resp.msg['target']
completer = DshellWordCompleter([
target
for target in needed_target
if target not in
values[dest].keys()
])
for c in completer.get_completions(
document, complete_event):
yield c
return
elif current_word in v:
completer = arg_def.get(
'completer', None)
for c in completer.get_completions(
document, complete_event):
if c.text not in v:
yield c
if len(v) == 0 or len(
v) == 1 and v[0] == finder:
return
k = None
v = None
# get source value
if k:
nargs = dest_args[k].get('nargs')
if nargs and not isinstance(nargs, int):
if k not in params and document.char_before_cursor == ' ':
if '--' not in words:
# next argument may be a positional parameter or an optional argument
# if nargs '+' means that at least 1 item must be provided
if not (nargs == '+' and v and len(v) -
1 == 0) and k not in params:
completer = DshellWordCompleter(
words=list(options.keys()))
for c in completer.get_completions(
document, complete_event):
yield c
if nargs == '*' or (nargs == '+' and v
and
len(v) - 1 > 0):
yield Completion('--')
else:
for p in params:
if 'choices' in params[p]:
completer = DshellWordCompleter(
params[p].get("choices"))
elif 'completer' in params[p]:
completer = params[p].get(
'completer')
for c in completer.get_completions(
document, complete_event):
yield c
break
elif k in params and document.char_before_cursor == ' ':
completer = DshellWordCompleter(
words=list(options.keys()))
for c in completer.get_completions(
document, complete_event):
yield c
# cursor is in params (not option) it may set an optional parameter
elif k in params and '--' not in words:
completer = DshellWordCompleter(
words=list(options.keys()))
for c in completer.get_completions(
document, complete_event):
yield c
if k in dest_args:
if 'choices' in dest_args[k]:
completer = DshellWordCompleter(
dest_args[k].get("choices"))
for c in completer.get_completions(
document, complete_event):
if (v and c.text
not in v) or v is None:
yield c
completer = dest_args[k].get('completer', None)
else:
completer = None
if completer:
if isinstance(completer, ResourceCompleter):
kwargs = dict(var_filters=values)
else:
kwargs = {}
for c in completer.get_completions(
document, complete_event, **kwargs):
if (v and c.text not in v) or v is None:
yield c
else:
completer = DshellWordCompleter(
words=list(options.keys()))
for c in completer.get_completions(
document, complete_event):
yield c
for p in params:
if getattr(namespace, p) is None:
if 'choices' in params[p]:
completer = DshellWordCompleter(
params[p].get("choices"))
elif 'completer' in params[p]:
completer = params[p].get('completer')
for c in completer.get_completions(
document, complete_event):
yield c
# No space in the input: behave exactly like `WordCompleter`.
else:
completer = DshellWordCompleter(list(self.options.keys()),
ignore_case=self.ignore_case)
for c in completer.get_completions(document, complete_event):
yield c
except Exception as e:
dprint(e)