def connect(self):
fn_server_public = os.path.expanduser(constants.FN_SERVER_CERT_PUBLIC)
xt_server_cert = self.config.get_vault_key("xt_server_cert")
use_public_half = False # cannot get the public half approach to work
try:
# write CERT file JIT
if use_public_half:
_, public_half = xt_server_cert.split("END PRIVATE KEY-----\n")
file_utils.write_text_file(fn_server_public, public_half)
else:
file_utils.write_text_file(fn_server_public, xt_server_cert)
self.conn = rpyc.ssl_connect(self.ip,
port=self.port,
keyfile=None,
certfile=fn_server_public)
if self.conn:
# magic step: allows our callback to work correctly!
# this must always be executed (even if self.conn is already true)
self.bgsrv = rpyc.BgServingThread(self.conn)
console.diag(" now running BgServingThread")
finally:
# delete the CERT file
#os.remove(fn_server_public)
pass
connected = not (not self.conn)
return connected
def set_timer(timeout):
console.print("set_timer called: timeout=", self.timeout)
time.sleep(self.timeout)
console.diag("timer triggered!")
plt.close("all")
print("closed all plots and the fig")
def sync_run_ssh(caller,
box_addr,
box_cmd,
report_error=True,
capture_output=True,
capture_as_bytes=False):
'''
This can be used to execute a cmd on the remote box, provided that an XT key
has previously been sent to the box with the 'xt keysend' command.
Note:
- we use the "-i <file>" option to send public half of keypair (avoid entering password)
- we use the "-v" (verbose) option to avoid a "quick edit" mode non-responsive issue (avoid ENTER key pressing)
'''
ssh_parts = make_ssh_cmd_parts(box_addr, box_cmd, capture_output,
capture_as_bytes)
console.diag(" running SSH: " + " ".join(ssh_parts))
exit_code, output = sync_run(ssh_parts,
report_error=report_error,
capture_output=capture_output,
capture_as_bytes=capture_as_bytes)
console.diag(" <script completed OK>")
return exit_code, output
def get_secret(name):
name = correct_name(name)
secrets = {}
if os.path.exists(FN_SECRETS):
text = file_utils.read_text_file(FN_SECRETS)
secrets = json.loads(text)
value = secrets[name] if name in secrets else None
console.diag("get_secret: name={}, value={}".format(name, value))
return value
def sync_run(cmd_parts,
capture_output=True,
shell=False,
report_error=False,
env_vars=None,
capture_as_bytes=False):
''' this does a synchronous run of the specified cmd/app and returns the app's exitcode. It runs
in the current working directory, but target app MUST be a fully qualified path. '''
universal_newlines = False
#cmd = " ".join(cmd_parts) if isinstance(cmd_parts, list) else cmd_parts
console.diag("sync_run: {}".format(cmd_parts))
# linux won't accept a command, only cmd parts
#assert isinstance(cmd_parts, (list, tuple))
if isinstance(cmd_parts, str):
cmd_parts = cmd_parts.split(" ")
if capture_output:
process = subprocess.run(cmd_parts,
cwd=".",
env=env_vars,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=universal_newlines,
shell=shell)
output = process.stdout
if not capture_as_bytes:
if not universal_newlines:
# since universal_newlines=False, we need to map bytes to str
output = output.decode("utf-8",
errors='backslashreplace').replace(
'\r', '')
output = filter_out_verbose_lines(output)
else:
process = subprocess.run(cmd_parts, cwd=".", env=env_vars, shell=shell)
output = None
exit_code = process.returncode
if report_error and exit_code:
console.print(output)
raise Exception("sync run failed, exit code={}, error={}".format(
exit_code, output))
return exit_code, output
def set_secret(name, value):
name = correct_name(name)
console.diag("set_secret: name={}, value={}".format(name, value))
file_utils.ensure_dir_exists(file=FN_SECRETS)
secrets = {}
# read existing secrets, if any
if os.path.exists(FN_SECRETS):
text = file_utils.read_text_file(FN_SECRETS)
secrets = json.loads(text)
secrets[name] = value
# write updates secrets
text = json.dumps(secrets)
file_utils.write_text_file(FN_SECRETS, text)
def scp_copy_file_to_box(caller,
box_addr,
fn_local,
box_fn,
report_error=True):
#cmd = 'scp -i {} "{}" {}:{}'.format(constants.LOCAL_KEYPAIR_PRIVATE, fn_local, box_addr, box_fn)
cmd_parts = [
"scp", "-i",
os.path.expanduser(constants.LOCAL_KEYPAIR_PRIVATE), fn_local,
"{}:{}".format(box_addr, box_fn)
]
console.diag(" copying script to box; cmd={}".format(cmd_parts))
exit_code, output = sync_run(cmd_parts)
if report_error and exit_code:
console.print(output)
raise Exception("scp copy command failed: {}".format(output))
return exit_code, output
def _load_grok_creds(self):
fn_keys = "keys.bin"
loaded = False
if not os.path.exists(fn_keys):
fn_keys = os.path.expanduser("~/.xt/teams/{}/keys.bin".format(
self.team_name))
if os.path.exists(fn_keys):
# GROK server creds
creds = file_utils.read_text_file(fn_keys)
self.apply_creds(creds)
fn_cert = os.path.join(os.path.dirname(fn_keys), "xt_cert.pem")
if os.path.exists(fn_cert):
cert = file_utils.read_text_file(fn_keys)
self.keys["xt_server_cert"] = cert
console.diag("init_creds: using grok server 'keys.bin' file")
loaded = True
return loaded
def read_file(self, fn, start_offset, end_offset):
fn = file_utils.fix_slashes(fn, is_linux=True)
# # leverage the read_file() function in psm.py
# ssh_cmd = "cd ~/.xt/cwd; python -c 'import psm; psm.read_file(\"{}\", {}, {})'" \
# .format(fn, start_offset, end_offset)
# error_code, read_bytes = process_utils.sync_run_ssh(None, self.box_addr, ssh_cmd, capture_as_bytes=True, report_error=False)
new_bytes = b""
try:
with self.ftp_client.file(fn) as infile:
infile.seek(start_offset)
if end_offset:
new_bytes = infile.read(end_offset - start_offset)
else:
new_bytes = infile.read()
except BaseException as ex:
console.diag("exception: ex={}".format(ex))
#new_bytes = read_bytes if not error_code else b""
return new_bytes
def plot_data(self, data_frames_by_cols):
console.diag("starting to plot data")
# on-demand import for faster XT loading
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib as mpl
import pylab
if not self.show_toolbar:
# hide the ugly toolbar at bottom left of plot window
mpl.rcParams['toolbar'] = 'None'
# apply seaborn styling
if self.style:
sns.set_style(self.style)
# decide how layout, titles, etc. will be set
group_names = set()
# gather group names (usually all are in the first dataset, but not always)
for dfx in data_frames_by_cols.values():
name_list = dfx[self.group_by].unique()
group_names.update(set(name_list))
group_names = list(group_names)
# this will sort the group names in a number-smart way
group_names.sort(key=utils.natural_keys)
group_count = len(group_names)
col_count = len(self.col_names)
break_on_groups = self.break_on and ("run" in self.break_on
or "group" in self.break_on)
break_on_cols = self.break_on and "col" in self.break_on
if break_on_groups and break_on_cols:
plot_count = group_count * col_count
elif break_on_groups:
plot_count = group_count
elif break_on_cols:
plot_count = col_count
else:
plot_count = 1
# calc true layout
if self.layout:
plot_rows, plot_cols = self.calc_actual_layout(
plot_count, self.layout)
else:
plot_cols = plot_count
plot_rows = 1
runs_per_plot = 1 if break_on_groups else group_count
cols_per_plot = 1 if break_on_cols else col_count
if runs_per_plot == 1:
plot_title = "$run"
legend_text = "$col"
elif cols_per_plot == 1:
plot_title = "$col"
legend_text = "$run"
else:
plot_title = None
legend_text = "$col ($run)"
if not self.plot_titles and plot_title:
self.plot_titles = [plot_title]
if not self.legend_titles:
self.legend_titles = [legend_text]
# configure matplotlib for our subplots
sharex = True
sharey = True
#plt.close()
window_size = (14, 6)
fig, plots = plt.subplots(plot_rows,
plot_cols,
figsize=window_size,
sharex=sharex,
sharey=sharey,
constrained_layout=True)
if not isinstance(plots, np.ndarray):
# make it consistent with plot_count > 1 plots
plots = [plots]
elif plot_rows > 1:
plots = plots.flatten()
fig.suptitle(self.title, fontsize=16)
if self.timeout:
# build a thread to close our plot window after specified time
from threading import Thread
def set_timer(timeout):
console.print("set_timer called: timeout=", self.timeout)
time.sleep(self.timeout)
console.diag("timer triggered!")
plt.close("all")
print("closed all plots and the fig")
thread = Thread(target=set_timer, args=[self.timeout])
thread.daemon = True # mark as background thread
thread.start()
line_index = 0
plot_index = 0
trace_count = 0
x_label = self.x_label if self.x_label else self.x_col
if self.aggregate == "none":
self.aggregate = None
if (self.aggregate and (break_on_cols and not break_on_groups)) \
or ((not self.aggregate) and break_on_cols):
# columns needs to be the outer loop
for c, col in enumerate(self.col_names):
if trace_count >= self.max_traces:
break
if c and break_on_cols:
plot_index += 1
line_index = 0
for r, group_name in enumerate(group_names):
if trace_count >= self.max_traces:
break
if r and break_on_groups:
plot_index += 1
line_index = 0
# PLOT MIDDLE
ax = plots[plot_index] # .gca()
self.plot_middle(data_frames_by_cols, ax, group_name, col,
self.x_col, x_label, line_index)
line_index += 1
trace_count += 1
else:
# run will work as the outer loop
for r, group_name in enumerate(group_names):
if trace_count >= self.max_traces:
break
if r and break_on_groups:
plot_index += 1
line_index = 0
for c, col in enumerate(self.col_names):
if trace_count >= self.max_traces:
break
if c and break_on_cols:
plot_index += 1
line_index = 0
# PLOT MIDDLE
ax = plots[plot_index] #.gca()
self.plot_middle(data_frames_by_cols, ax, group_name, col,
self.x_col, x_label, line_index)
line_index += 1
trace_count += 1
if self.save_to:
plt.savefig(self.save_to)
if self.show_plot:
pylab.show()