async def get_log(ctx, logfile):
c = 'ssh %s@%s ls HomeAlone/code/logs/web/' % (HOSTN, SERVE)
if logfile in utils.cmd(c, False):
await ctx.send('Found %s. Downloading it now.' % logfile)
c = f"sftp {HOSTN}@{SERVE}:/home/{HOSTN}/HomeAlone/code/logs/web/ <<< $'get {logfile}'"
utils.arr2str(utils.cmd(c, False))
await ctx.send('Finished downloading**[%d bytes]**' %
os.path.getsize(os.getcwd() + '/%s' % logfile))
def query_shares(self, csock, caddr, api_req):
file_list = os.listdir(os.getcwd() + '/PoolData/Shares')
msg = 'Shared Files on %s@%s:\n' % (self.info['host'],
self.info['internal'])
msg += utils.arr2str(file_list)
csock.send(msg)
return csock
def test(self, loader: DataLoader, logdir: Path):
group = logdir.name.split('_')[0]
self.writer = CustomWriter(str(logdir), group=group)
avg_measure = None
self.model.eval()
pbar = tqdm(loader, desc=group, dynamic_ncols=True)
for i_iter, data in enumerate(pbar):
# get data
x, y = self._pre(data, loader.dataset) # B, C, T
T_ys = data['T_ys']
# forward
output = self.model(x) # [..., :y.shape[-1]]
# write summary
one_sample = MulchWavDataset.decollate_padded(data, 0) # F, T, C
out_one = self._post_one(output, T_ys, 0, loader.dataset.norm_out)
measure = self.writer.write_one(i_iter, **out_one, **one_sample)
if avg_measure is None:
avg_measure = measure
else:
avg_measure += measure
# print
str_measure = arr2str(measure).replace('\n', '; ')
pbar.write(str_measure)
self.model.train()
avg_measure /= len(loader.dataset)
self.writer.add_text(f'{group}/Average Measure/Proposed',
str(avg_measure[0]))
self.writer.add_text(f'{group}/Average Measure/Reverberant',
str(avg_measure[1]))
self.writer.close() # Explicitly close
print()
str_avg_measure = arr2str(avg_measure).replace('\n', '; ')
print(f'Average: {str_avg_measure}')
async def list_log_files(ctx):
await ctx.send('*Getting list of log files*')
c = 'ssh %s@%s ls -la HomeAlone/code/logs/web/' % (HOSTN, SERVE)
result = '```' + (utils.arr2str(utils.cmd(c, False))) + '```'
try:
await ctx.send(result)
except:
print(result)
pass
async def check_alarm(ctx, filename, n):
print('[-] Checking Alarm File')
while ARMED:
await asyncio.sleep(35)
try:
c = f"sftp {HOSTN}@{SERVE}:/home/{HOSTN}/TripWire/tripwire/.alerts/alarm/ <<< $'get alarm'"
utils.arr2str(utils.cmd(c, False))
if filename in utils.swap(filename, True):
n = N
m = '{0.author.mention} **New Connection <a:siren:833794872204722248> **'.format(
ctx.message)
m += '```' + utils.arr2str(
utils.cmd(f"tail -n 3 {filename} ", False)) + '```'
await ctx.send(m)
except IndexError:
print('[!] Unable to read log file')
pass
def validate(self, loader: DataLoader, logdir: Path, epoch: int):
""" Evaluate the performance of the model.
:param loader: DataLoader to use.
:param logdir: path of the result files.
:param epoch:
"""
self.model.eval()
avg_loss = torch.zeros(len(self.criterions), device=self.out_device)
pbar = tqdm(loader,
desc='validate ',
postfix='[0]',
dynamic_ncols=True)
for i_iter, data in enumerate(pbar):
# get data
x, y = self._pre(data, loader.dataset) # B, C, F, T
T_ys = data['T_ys']
# forward
output = self.model(x) # [..., :y.shape[-1]]
# loss
loss = self._calc_loss(y, output, T_ys)
avg_loss += loss
# print
loss_np = loss.cpu().numpy() / len(T_ys)
pbar.set_postfix_str(arr2str(loss_np, ndigits=1))
# write summary
if i_iter == 0:
out_one = self._post_one(output, T_ys, 0,
loader.dataset.norm_out)
if not self.writer.reused_sample:
one_sample = MulchWavDataset.decollate_padded(data, 0)
else:
one_sample = dict()
self.writer.write_one(epoch, **out_one, **one_sample)
avg_loss /= len(loader.dataset)
tag = 'loss/valid'
self.writer.add_scalar(tag, avg_loss.sum().item(), epoch)
if len(self.criterions) > 1:
for idx, (n, ll) in enumerate(zip(hp.criterion_names, avg_loss)):
self.writer.add_scalar(f'{tag}/{idx + 1}. {n}', ll.item(),
epoch)
self.model.train()
return avg_loss
async def set_alarm(ctx, filename):
try:
c = f"sftp {HOSTN}@{SERVE}:/home/{HOSTN}/HomeAlone/code/logs/web/ <<< $'get {filename}'\n"
c += f"ssh {HOSTN}@{SERVER} echo '{PATH}' >> /home/{HOSTN}/HomeAlone/code/filelist.txt\n"
utils.arr2str(utils.cmd(c, False))
PATH = f'/home/{HOSTN}/HomeAlone/code/logs/web/{filename}'
utils.cmd(c2)
n = int(
utils.cmd("cat %s| grep 'Connection at ' | wc -l" % filename,
False).pop())
await ctx.send(
':ok_hand: *Setting Alarm on %s*, which currently has **%d** entries.'
% (filename, n))
ARMED = True
bot.loop.create_task(check_alarm(ctx, filename, n))
except:
c = 'ssh %s@%s ls -la HomeAlone/code/logs/web/' % (HOSTN, SERVE)
result = 'Something went wrong... Select one of these to set alarm on:\n'
result += '```' + (utils.arr2str(utils.cmd(c, False))) + '```'
pass
async def show_connection(ctx):
msg = 'Aw Geez, lets see who is connected <:morty:833787213766066176>'.format(
ctx.message)
await ctx.send(msg)
c = 'ssh %s@%s netstat -antup' % (HOSTN, SERVE)
result = '```' + (utils.arr2str(utils.cmd(c, False))) + '```'
try:
await ctx.send(result)
except:
print(result)
await ctx.send('```' + result.split('established)\n')[1:])
pass
async def read_log(ctx, filename):
c = f'ssh {HOSTN}@{SERVE} ls HomeAlone/code/logs/web/'
if filename in utils.cmd(c, False):
await ctx.send('**This will take a minute...**')
await ctx.send('<a:rickspin:834261749846507520>')
c = f"sftp {HOSTN}@{SERVE}:/home/{HOSTN}/HomeAlone/code/logs/web/ <<< $'get {filename}'"
utils.arr2str(utils.cmd(c, False))
f = open(filename, 'r')
while True:
piece = f.read(1024)
if not piece:
break
try:
await ctx.send('```\n' + piece.replace(SERVE, '<removed>') +
'\n```')
time.sleep(3)
except:
print(result)
pass
f.close()
os.remove(filename)
else:
await ctx.send("I can't find %s" % filename)
def check_messaging_nodes(self):
fn = 'PoolData/Config/Channels/Self/messaging.json'
if os.path.isfile(fn):
with open('PoolData/Config/Channels/Self/messaging.json',
'r') as f:
messaging = json.load(f)
if 'master' in messaging.keys():
print('[*] Master node is: %s' % messaging['master'])
if 'brokers' in messaging.keys():
print('[*] Brokers are:\n%s' %
utils.arr2str('-'.join(messaging['brokers'])))
# check if backend was killed
if 'listening' in messaging.keys():
self.running = messaging['listening']
if not self.running:
print('[!!] Triggering Shutdown from Backend')
def main():
if '--setup' in sys.argv:
utils.initialize_folders()
utils.add_local_peers_pv2()
else:
nodes = test_connections(False)
if '--cmd' in sys.argv and len(sys.argv) > 2:
hname = utils.sys.argv[2]
cmd = utils.arr2chr(sys.argv[3:])
if hname in nodes.keys():
i = nodes[hname]['ip']
h = nodes[hname]['hname']
p = nodes[hname]['pword']
result = utils.arr2str(utils.ssh_exec(cmd, i, h, p, False))
print result
def test(self, loader: DataLoader, logdir: Path):
group = logdir.name.split('_')[0]
self.writer = CustomWriter(str(logdir), group=group)
avg_measure = None
self.model.eval()
pbar = tqdm(loader, desc=group, dynamic_ncols=True)
for i_iter, data in enumerate(pbar):
# get data
x, y = self.preprocess(data) # B, C, F, T
T_ys = data['T_ys']
# forward
output = self.model(x) # [..., :y.shape[-1]]
# write summary
one_sample = DirSpecDataset.decollate_padded(data, 0) # F, T, C
out_one = self.postprocess(output, T_ys, 0, loader.dataset)
DirSpecDataset.save_dirspec(
logdir / hp.form_result.format(i_iter),
**one_sample, **out_one
)
measure = self.writer.write_one(i_iter, **out_one, **one_sample)
if avg_measure is None:
avg_measure = AverageMeter(init_value=measure, init_count=len(T_ys))
else:
avg_measure.update(measure)
# print
# str_measure = arr2str(measure).replace('\n', '; ')
# pbar.write(str_measure)
self.model.train()
avg_measure = avg_measure.get_average()
self.writer.add_text(f'{group}/Average Measure/Proposed', str(avg_measure[0]))
self.writer.add_text(f'{group}/Average Measure/Reverberant', str(avg_measure[1]))
self.writer.close() # Explicitly close
print()
str_avg_measure = arr2str(avg_measure).replace('\n', '; ')
print(f'Average: {str_avg_measure}')
async def scan_host(ctx, ip):
await ctx.send('<a:radar:794818374420529162> *Scanning* **%s**' % ip)
c = 'nmap -sV %s' % ip
result = '```' + (utils.arr2str(utils.cmd(c, False))) + '```'
await ctx.send(result)
def list_commands(self, csock, caddr, api_req):
cmds = self.actions.keys()
csock.send('Commands:\n%s' % utils.arr2str(cmds))
return csock
def train(self,
loader_train: DataLoader,
loader_valid: DataLoader,
logdir: Path,
first_epoch=0):
n_train_data = len(loader_train.dataset)
# Learning Rates Scheduler
scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(
self.optimizer, last_epoch=first_epoch - 1, **hp.scheduler)
avg_loss = torch.zeros(len(self.criterions), device=self.out_device)
self.writer = CustomWriter(str(logdir),
group='valid',
purge_step=first_epoch)
# self.writer.add_graph(
# self.model.module if isinstance(self.model, nn.DataParallel) else self.model,
# torch.zeros(2, *hp.dummy_input_size),
# # operator_export_type='RAW',
# )
# Start Training
for epoch in range(first_epoch, hp.n_epochs):
print()
pbar = tqdm(loader_train,
desc=f'epoch {epoch:3d}',
postfix='[]',
dynamic_ncols=True)
for i_iter, data in enumerate(pbar):
# get data
x, y = self._pre(data, loader_train.dataset) # B, C, T
T_ys = data['T_ys']
i_first = 0
loss_t = None
for t in range(0, y.shape[-1], hp.l_target):
self.optimizer.zero_grad()
while i_first < y.shape[0] and t >= T_ys[i_first]:
i_first += 1
seg_y = y[i_first:, :, t:t + hp.l_target]
if seg_y.shape[-1] < 5:
break
seg_x = x[i_first:, :, t:t + hp.l_input]
seg_T_ys = np.clip(T_ys[i_first:] - t,
a_min=None,
a_max=hp.l_target)
# forward
output = self.model(seg_x)[..., :
seg_y.shape[-1]] # B, C, T
loss_t = self._calc_loss(seg_y, output, seg_T_ys)
loss_t_sum = loss_t.sum()
# backward
loss_t_sum.backward()
self.optimizer.step()
# print
avg_loss += loss_t.detach_()
del loss_t_sum, seg_x, seg_y, output
scheduler.step(epoch + i_iter * hp.batch_size / n_train_data)
loss_t_last_np = loss_t.cpu().numpy() / len(T_ys)
pbar.set_postfix_str(arr2str(loss_t_last_np, ndigits=1))
avg_loss /= n_train_data
tag = 'loss/train'
self.writer.add_scalar(tag, avg_loss.sum().item(), epoch)
if len(self.criterions) > 1:
for i, (n, ll) in enumerate(zip(hp.criterion_names, avg_loss)):
self.writer.add_scalar(f'{tag}/{i + 1}_{n}', ll.item(),
epoch)
# Validation
self.validate(loader_valid, logdir, epoch)
# save loss & model
if epoch % hp.period_save_state == hp.period_save_state - 1:
torch.save((
self.model.module.state_dict() if isinstance(
self.model, nn.DataParallel) else
self.model.state_dict(),
self.optimizer.state_dict(),
), logdir / f'{hp.model_name}_{epoch}.pt')
self.writer.close()
def test(self, loader: DataLoader, logdir: Path):
def save_forward(module: nn.Module, in_: Tensor, out: Tensor):
""" save forward propagation data
"""
module_name = str(module).split('(')[0]
dict_to_save = dict()
# dict_to_save['in'] = in_.detach().cpu().numpy().squeeze()
dict_to_save['out'] = out.detach().cpu().numpy().squeeze()
i_module = module_counts[module_name]
for i, o in enumerate(dict_to_save['out']):
save_forward.writer.add_figure(
f'{group}/blockout_{i_iter}/{module_name}{i_module}',
draw_spectrogram(o, to_db=False),
i,
)
scio.savemat(
str(logdir / f'blockout_{i_iter}_{module_name}{i_module}.mat'),
dict_to_save,
)
module_counts[module_name] += 1
group = logdir.name.split('_')[0]
self.writer = CustomWriter(str(logdir), group=group)
avg_measure = None
self.model.eval()
# register hook to save output of each block
module_counts = None
if hp.n_save_block_outs:
module_counts = defaultdict(int)
save_forward.writer = self.writer
if isinstance(self.model, nn.DataParallel):
module = self.model.module
else:
module = self.model
for sub in module.children():
if isinstance(sub, nn.ModuleList):
for m in sub:
m.register_forward_hook(save_forward)
elif isinstance(sub, nn.ModuleDict):
for m in sub.values():
m.register_forward_hook(save_forward)
else:
sub.register_forward_hook(save_forward)
pbar = tqdm(loader, desc=group, dynamic_ncols=True)
for i_iter, data in enumerate(pbar):
# get data
x, y = self.preprocess(data) # B, C, F, T
T_ys = data['T_ys']
# forward
if module_counts is not None:
module_counts = defaultdict(int)
if 0 < hp.n_save_block_outs == i_iter:
break
output = self.model(x) # [..., :y.shape[-1]]
# write summary
one_sample = DirSpecDataset.decollate_padded(data, 0) # F, T, C
out_one = self.postprocess(output, T_ys, 0, loader.dataset)
# DirSpecDataset.save_dirspec(
# logdir / hp.form_result.format(i_iter),
# **one_sample, **out_one
# )
measure = self.writer.write_one(
i_iter,
eval_with_y_ph=hp.eval_with_y_ph,
**out_one,
**one_sample,
)
if avg_measure is None:
avg_measure = AverageMeter(init_value=measure,
init_count=len(T_ys))
else:
avg_measure.update(measure)
self.model.train()
avg_measure = avg_measure.get_average()
self.writer.add_text('Average Measure/Proposed', str(avg_measure[0]))
self.writer.add_text('Average Measure/Reverberant',
str(avg_measure[1]))
self.writer.close() # Explicitly close
print()
str_avg_measure = arr2str(avg_measure).replace('\n', '; ')
print(f'Average: {str_avg_measure}')
async def pull_pcap(ctx):
c = f"sftp {HOSTN}@{SERVE}:/home/{HOSTN}/ <<< $'get honey.pcap'"
reply = utils.arr2str(utils.cmd(c, False))
await ctx.send("'''%s'''" % reply)
async def start_tcpdump(ctx):
# c = 'iface=$(ip route get 1.1.1.1 | awk {print $5; exit});'
# c+= f'tcpdump -ne -i $iface -Q in host {SERVE} and port 8080 -w honey.pcap'
c = f"ssh {HOSTN}@{SERVE} bash listen.sh &"
reply = utils.arr2str(utils.cmd(c, False))
await ctx.send("'''%s'''" % reply)
def query_peerlist(self, csock, caddr, api_req):
msg = 'Peer List:\n'
msg += utils.arr2str(self.peers)
csock.send(msg)
return csock
async def kill_tcpdump(ctx):
c = f"ssh {HOSTN}@{SERVE} kill -9 $(pidof tcpdump)"
reply = utils.arr2str(utils.cmd(c, False))
await ctx.send("'''%s'''" % reply)
def test(self, loader: DataLoader, logdir: Path):
def save_forward(module: nn.Module, in_: Tensor, out: Tensor):
module_name = str(module).split('(')[0]
dict_to_save = dict()
# dict_to_save['in'] = in_.detach().cpu().numpy().squeeze()
dict_to_save['out'] = out.detach().cpu().numpy().squeeze()
i_module = module_counts[module_name]
for i, o in enumerate(dict_to_save['out']):
save_forward.writer.add_figure(
f'{group}/blockout_{i_iter}/{module_name}{i_module}',
draw_spectrogram(o, to_db=False),
i,
)
scio.savemat(
str(logdir / f'blockout_{i_iter}_{module_name}{i_module}.mat'),
dict_to_save,
)
module_counts[module_name] += 1
group = logdir.name.split('_')[0]
self.writer = CustomWriter(str(logdir), group=group)
avg_measure = None
self.model.eval()
module_counts = None
if hp.n_save_block_outs:
module_counts = defaultdict(int)
save_forward.writer = self.writer
for sub in self.module.children():
if isinstance(sub, nn.ModuleList):
for m in sub:
m.register_forward_hook(save_forward)
elif isinstance(sub, nn.ModuleDict):
for m in sub.values():
m.register_forward_hook(save_forward)
else:
sub.register_forward_hook(save_forward)
pbar = tqdm(loader, desc=group, dynamic_ncols=True)
cnt_sample = 0
for i_iter, data in enumerate(pbar):
# get data
x, mag, max_length, y = self.preprocess(data) # B, C, F, T
T_ys = data['T_ys']
# forward
if module_counts is not None:
module_counts = defaultdict(int)
if 0 < hp.n_save_block_outs == i_iter:
break
_, output, residual = self.model(x,
mag,
max_length,
repeat=hp.repeat_test)
# write summary
for i_b in range(len(T_ys)):
i_sample = cnt_sample + i_b
one_sample = ComplexSpecDataset.decollate_padded(
data, i_b) # F, T, C
out_one = self.postprocess(output, residual, T_ys, i_b,
loader.dataset)
ComplexSpecDataset.save_dirspec(
logdir / hp.form_result.format(i_sample), **one_sample,
**out_one)
measure = self.writer.write_one(i_sample,
**out_one,
**one_sample,
suffix=f'_{hp.repeat_test}')
if avg_measure is None:
avg_measure = AverageMeter(init_value=measure)
else:
avg_measure.update(measure)
# print
# str_measure = arr2str(measure).replace('\n', '; ')
# pbar.write(str_measure)
cnt_sample += len(T_ys)
self.model.train()
avg_measure = avg_measure.get_average()
self.writer.add_text(f'{group}/Average Measure/Proposed',
str(avg_measure[0]))
self.writer.add_text(f'{group}/Average Measure/Reverberant',
str(avg_measure[1]))
self.writer.close() # Explicitly close
print()
str_avg_measure = arr2str(avg_measure).replace('\n', '; ')
print(f'Average: {str_avg_measure}')
def show_methods(self, cs, ca, req):
result = utils.arr2str('-'.join(self.actions.keys()))
cs.send(result)
cs.close()