def keyword_count(row):
key_match = {}
out_row = []
match_row = []
row_index = None
for cell in row:
if isinstance(cell, EmptyCell):
continue
value = cell.value
if not isinstance(value, unicode) and not isinstance(value, str):
continue
if cell.column_letter in cfg.output_columns:
out_row.append(value)
for key in cfg.keywords:
if key in value:
utils.print_stdout('found keyword [' + key + '] in [' +
cell.coordinate + ']')
key_match[key] = value
row_index = cell.row
break
# last = key_match.get(key, None)
# if not last:
# key_match[key] = value
# else:
# if last != value:
# match_row.append(last)
# match_row.append(value)
# row_index = cell.row
# break
if len(key_match) > 1:
utils.print_stdout('output row [' + str(row_index) + ']')
return out_row + key_match.values()
return []
def ssh_client(rank):
port = 8080
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print(f'process {rank} connecting...\n')
client.connect(hostname=utils.ADDRESS,
port=utils.PORT,
username=USERNAME,
password=PASSWORD)
command = ''
if rank == 1:
# first, set up a server that prints out the password to the client
command = f'echo {PASSWORD} | nc -lp {port}'
else:
# execute the client to get the password from the server
# if the password is correct, then send the password to the server
command = f'./suconnect {port}'
print(f"process {rank}: {command}\n")
_, stdout, _ = client.exec_command(command, timeout=5, get_pty=True)
stdout = stdout.readlines()
utils.print_stdout(stdout)
if rank == 1:
# bandit21 password: gE269g2h3mw3pwgrj0Ha9Uoqen1c9DGr
print(f"bandit21 password: {utils.get_password(stdout)}")
client.close()
def main():
utils.print_stdout('Load Config...')
read_path = cfg.read_path
if not os.path.exists(read_path):
utils.print_stdout(
'Error: read_path[%s] is not exists, please check!' % read_path)
return
wb1 = Workbook()
out_sheet = wb1.active
utils.print_stdout('->start data loading... [%s]' % read_path)
from_sheet = load_workbook(filename=read_path, read_only=True).active
utils.print_stdout('Data Loading Done')
rows = from_sheet.rows
for row in rows:
res = keyword_count(row)
if len(res) > 1:
out_sheet.append(res)
wb1.save(cfg.output_path)
utils.print_stdout('Save result to Excel [%s]' % cfg.output_path)
FOLDERNAME = str(uuid.uuid4())
REPONAME = 'ssh://bandit29-git@localhost/home/bandit29-git/repo'
print(f'folder name: {FOLDERNAME}\n')
print(
f"mkdir /tmp/{FOLDERNAME} && cd /tmp/{FOLDERNAME} && git clone {REPONAME}")
stdin, stdout, _ = client.exec_command(
f"mkdir /tmp/{FOLDERNAME} && cd /tmp/{FOLDERNAME} && git clone {REPONAME}",
get_pty=True)
time.sleep(1)
stdin.write('yes\n')
time.sleep(1)
stdin.write(PASSWORD + '\n')
stdin.flush()
utils.print_stdout(stdout)
print(f"cat /tmp/{FOLDERNAME}/repo/README.md")
_, stdout, _ = client.exec_command(f"cat /tmp/{FOLDERNAME}/repo/README.md")
utils.print_stdout(stdout)
# from the above command, we can read README.md, which shows that
# there is no password in production branch or the master branch
# which might suggest the existence of a development branch
# reference: https://www.jquery-az.com/list-branches-git/
# to list all branches in local and remote repositories, we can use
# git branch -a
print(f"cd /tmp/{FOLDERNAME}/repo && git branch -a")
_, stdout, _ = client.exec_command(
f"cd /tmp/{FOLDERNAME}/repo && git branch -a")
utils.print_stdout(stdout)
z_points=batch.to(device),
dv=dv,
grid=Mxy,
model=net,
constant=None,
batch_size=batch_size,
metric_grid_sum=grid_metric_sum,
grid_sampled=Mxy_batch)
lpzs_test.append(lpz)
test_lpz_log[epoch] = torch.cat(lpzs_test).mean().item()
lpz_std_log[epoch] = torch.cat(lpzs).std().item()
utils.print_stdout('Epoch: {}, P(z) train: {:.4f}, P(z) test: {:.4f} \
, mu: [{:.4f},{:.4f}], std: {}'.format(epoch,
torch.cat(lpzs).mean().item(),
torch.cat(lpzs_test).mean().item(),
torch.cat(mus).mean(dim=0)[0].item(),
torch.cat(mus).mean(dim=0)[1].item(),
np.round(A.data.tolist(), 4)))
if epoch > 1:
if (test_lpz_log[epoch - 1] - 0.1 * lpz_std_log[epoch]) < test_lpz_log[epoch] < (
test_lpz_log[epoch - 1] + 0.1 * lpz_std_log[epoch]):
early_stopping_counter += 1
else:
early_stopping_counter = 0
if early_stopping_counter >= 5:
break
if test_lpz_log[epoch] < best_nll:
best_nll = test_lpz_log[epoch]
best_mu = mu.clone().detach()
def main():
read_path = cfg.read_path
check_column = cfg.check_column
utils.print_stdout('Load Config...')
if not os.path.exists(read_path):
utils.print_stdout('Error: read_path[%s] is not exists, please check!' % read_path)
return
wb1 = Workbook()
passenger_sheet = wb1.active
passenger_sheet.title = "passenger_car"
other_sheet = wb1.create_sheet(title="others")
if cfg.titles and len(cfg.titles) != 0:
utils.print_stdout("->setting titles...")
if len(cfg.titles) != len(cfg.output_columns)+len(cfg.output_keyword):
cfg.print_stdout('Error: the length of titles is different from output_column\'s, please check!')
return 1
passenger_sheet.append(cfg.titles)
other_sheet.append(cfg.titles)
if setTitleStyle:
for cell in passenger_sheet[1]:
cell.style = cfg.style_titleRow
for cell in other_sheet[1]:
cell.style = cfg.style_titleRow
utils.print_stdout('->start data loading... [%s]' % read_path)
wb2 = load_workbook(filename=read_path, read_only=True)
utils.print_stdout('Data Loading Done')
data = wb2.active
rows = data.rows
check_col = column_index_from_string(check_column)
utils.print_stdout('->start data filtering...')
for row in rows:
value = row[check_col - 1].value
if not isinstance(value, unicode) and not isinstance(value, str):
utils.print_stdout('value type is not unicode or str, so ignore value [%s]' % value)
continue
if value in cfg.match:
utils.print_stdout('value match [%s]' % value)
new_raw = format_row(row)
passenger_sheet.append(new_raw)
continue
if utils.cell_include_check(value, cfg.include) and utils.cell_exclude_check(value, cfg.exclude):
utils.print_stdout('value include and exclude [%s]' % value)
new_raw = format_row(row)
other_sheet.append(new_raw)
utils.print_stdout('Data Filtration Done')
wb1.save(filename=cfg.output_path)
utils.print_stdout('Save result to Excel [%s]' % cfg.output_path)
