def show_help():
global LOGGER, valid_commands
# Find the help key for this command
cmd = util.get_param(1).lower()
param1 = util.get_param(2).lower()
if cmd == "" or param1 == "":
# Missing required parameters!
p("}}rnMissing Required Parameters! " + cmd + " - " + param1 + "}}xx",
log_level=1)
return False
if not param1 in valid_commands:
p("}}rnInvalid Command! " + param1 + "}}xx", log_level=1)
commands = list(valid_commands.keys())
p("}}yn Valid Commands: " + str(commands) + "}}xx")
p("}}ybFor help - type mgmt.exe help (command)}}xx")
return False
cmd_parts = valid_commands[param1]
if cmd_parts is None:
p("}}rnInvalid Command - not configured! " + param1 + "}}xx",
log_level=1)
return False
help_msg = cmd_parts["help"]
if help_msg is None:
p("}}rnNo Help Provided! " + param1 + "}}xx", log_level=1)
return False
p("}}yb" + help_msg + "}}xx")
return True
def remove_nic():
# Get the params for the nic
nic_name = util.get_param(2)
nic_network = util.get_param(3)
if nic_name == "" or nic_network == "":
p("}}rbError - Invalid paramaters!}}xx")
return False
# Get the list of approved nics
# Load the value from the registry
try:
approved_nics_json = RegistrySettings.get_reg_value(
app="OPEService", value_name="approved_nics", default="[]")
nic_list = json.loads(approved_nics_json)
new_nic_list = []
# Loop through old list and add any that don't match remove parameters
removed = False
for item in nic_list:
#p("checking " + item[0] + " -> " + item[1])
if item[0] == nic_name and item[1] == nic_network:
#p("FOUND!")
removed = True
else:
# Not the one to remove, put it in the new_nic_list
new_nic_list.append(item)
if removed:
# Write this back to the registry
approved_nics_json = json.dumps(new_nic_list)
RegistrySettings.set_reg_value(app="OPEService",
value_name="approved_nics",
value=approved_nics_json)
p("}}gnRemoved " + nic_name + " on netowrk " + nic_network +
"}}xx",
log_level=1)
else:
p("}}rnNOT FOUND - NOT Removed " + nic_name + " on netowrk " +
nic_network +
" (Can't remove system pre-approved nic entries)}}xx")
except Exception as ex:
p("}}rbUnable to write approved nics to the registry!}}xx\n" +
str(ex),
log_level=1)
return False
# Force device list refresh
NetworkDevices.init_device_list(refresh=True)
return True
def __init__(self, task):
# Task (environment) information
self.task = task # should contain observation_space and action_space
self.state_size = np.prod(3)
self.state_range = self.task.observation_space.high[
0:3] - self.task.observation_space.low[0:3]
self.action_size = np.prod(3)
self.action_range = self.task.action_space.high[
0:3] - self.task.action_space.low[0:3]
# Policy parameters
self.w = np.random.normal(
size=(
3, 3
), # weights for simple linear policy: state_space x action_space self.state_size
scale=(self.action_range / (2 * self.state_size)).reshape(
1, -1)) # start producing actions in a decent range
# Score tracker and learning parameters
self.best_w = None
self.best_score = -np.inf
self.noise_scale = 0.1
self.episode_num = 1
self.stats_filename = os.path.join(
util.get_param('out'),
"stats_{}.csv".format(util.get_timestamp())) # path to CSV file
self.stats_columns = ['episode',
'total_reward'] # specify columns to save
# Episode variables
self.reset_episode_vars()
def export_group_policy():
ret = True
gpo_name = util.get_param(2, "exported_gpo")
# LGPO.exe is in the rc sub folder of the mgmt tool
app_folder = util.get_app_folder()
lgpo_path = os.path.join(app_folder, "rc")
gpo_path = os.path.join(lgpo_path, gpo_name)
# NOTE - lgpo needs full path to gpo export directory
cmd = "lgpo.exe /b \"" + gpo_path + "\""
# Remove the folder if it exists
p("Removing old files: " + gpo_path)
shutil.rmtree(gpo_path, ignore_errors=True)
# Folder has to exist
os.makedirs(gpo_path, exist_ok=True)
returncode, output = ProcessManagement.run_cmd(cmd,
cwd=lgpo_path,
attempts=5,
require_return_code=0,
cmd_timeout=15)
if returncode == -2:
# Unable to restore gpo?
p("}}rnERROR - Unable to dump group policy!}}xx\n" + output)
ret = False
return ret
p(output)
return ret
def delete_user(user_name=None):
if user_name is None:
user_name = util.get_param(2, None)
if user_name is None:
p("}}enInvalid User name - not removing account!}}xx")
return False
curr_user = None
try:
curr_user = accounts.principal(user_name)
except Exception as ex:
p("}}rbInvalid User Account - Not deleting!}}xx", debug_level=1)
return False
# Remove the profile first
ret = UserAccounts.remove_account_profile(user_name)
# Remove the local user
try:
curr_user.delete()
except Exception as ex:
p("}}rbError - Unable to remove account: " + str(user_name) +
"}}xx\n" + str(ex))
return False
return True
def apply_firewall_policy():
ret = True
if RegistrySettings.is_debug():
p("}}rbDEBUG MODE ON - Skipping apply firewall policy}}xx")
return True
policy_file_name = util.get_param(2, "firewall_config.wfw")
# Should be in RC sub folder under the app
app_folder = util.get_app_folder()
rc_path = os.path.join(app_folder, "rc")
policy_file_path = os.path.join(rc_path, policy_file_name)
# netsh advfirewall import "%~dp0rc\firewall_config.wfw" 2>NUL 1<NUL
cmd = "%SystemRoot%\\system32\\netsh advfirewall import \"" + policy_file_path + "\""
returncode, output = ProcessManagement.run_cmd(cmd,
attempts=5,
require_return_code=0,
cmd_timeout=15)
if returncode == -2:
# Error running command?
p("}}rbError - Unable to reset firewall back to defaults?}}xx\n" +
output)
ret = False
return ret
def __init__(self, dataset, model, center_loader=None,
param=None):
self._dataset = dataset
self._center_loader = center_loader
proto_name, model_name = util.get_model(dataset, model)
self._fx, self._fy, self._ux, self._uy = util.get_param(dataset) if param is None else param
self._net = caffe.Net(proto_name, caffe.TEST, weights=model_name)
self._input_size = self._net.blobs['data'].shape[-1]
self._cube_size = 150
def save_log(mode="w"):
with open(log_path, mode) as outFile:
if mode == "a":
print("----------resume training/early stopping ---------",
file=outFile)
else:
print("-----------Tokenizer---------", file=outFile)
# commands for later training
long, short = util.get_command(str(args))
p = "python Tokenizer.py "
long = p + long
short = p + short
print(">>> command with full parameters", file=outFile)
print(long, file=outFile)
print("\n>>> command with short parameters", file=outFile)
print(short, file=outFile)
print("", file=outFile)
# model parameters and losses
print("file name = ", args.save_to, file=outFile)
print("", file=outFile)
print(">>> ", args.add_note, file=outFile)
print("bi_lstm ", bi_lstm, file=outFile)
print("train classifier for " + str(args.epoch) + " epoch",
file=outFile)
print("trainLosses ", trainLosses, file=outFile)
print("devLosses ", devLosses, file=outFile)
print("", file=outFile)
print("count train sample ", count_train_samples, file=outFile)
print("count dev sample ", count_dev_samples, file=outFile)
# model parameters
print("", file=outFile)
l = str(args)[10:].strip(")").split(",")
for i in l:
print(i, file=outFile)
print("", file=outFile)
# data set
print("train set: ", train_path, file=outFile)
print("dev set: ", dev_path, file=outFile)
print("test set: ", test_path, file=outFile)
print("", file=outFile)
# number of samples
print("count train sample ", count_train_samples, file=outFile)
print("count dev sample ", count_dev_samples, file=outFile)
# parameter flags (used when later import this model to another model)
print("config for later download : ", file=outFile)
p = util.get_param(str(args))
print(p, file=outFile)
print("", file=outFile)
def lock_screen_for_user(user_name=None):
# Find the user in question and lock the workstation
if user_name is None:
user_name = util.get_param(2, None)
if user_name is None:
# Lock for the current user if no name
return UserAccounts.lock_screen_for_current_user()
p("}}ybLocking screen for other users - Not Implemented Yet!}}xx")
# TODO - lock_workstation
# Lookup the user specified and run this under their account
return False
def ping_smc(smc_url=None):
# See if we can bounce off the SMC server and get a response
if smc_url is None:
# Try and get from command line
smc_url = util.get_param(2, None)
if smc_url is None:
# Nothing on command line? Get from registry
smc_url = RegistrySettings.get_reg_value(value_name="smc_url", default="https://smc.ed")
force = util.get_param(3, "")
if force == "-f":
force = True
else:
force = False
if not force and not CredentialProcess.is_time_to_ping_smc():
#p("}}gnNot time to ping smc, skipping...}}xx", log_level=4)
return True
RegistrySettings.set_reg_value(value_name="last_smc_ping_time", value=time.time())
if not RestClient.ping_smc(smc_url):
p("}}mnNot able to ping SMC " + smc_url + "}}xx", log_level=4)
# Ok to return true - we just don't do more maintenance
return True
# We are connected, do maintenance
# Check if time to sync time
SystemTime.sync_time_w_ntp()
# Check if time to sync stuff (lms app, folders, logs)...
# TODO
# Check if time to auto upgrade
CredentialProcess.start_upgrade_process()
return True
def save_csv(f="LM_log.csv"):
csv_path = os.path.join(CHECKPOINTS_LM, f)
with open(csv_path, "a+") as table:
print(args.save_to, file=table, end=';')
print(args.dataset, file=table, end=';')
print(num_epoch + 1, file=table, end=';')
print("trainLosses ", trainLosses, file=table, end=';')
print("devLosses ", devLosses, file=table, end=';')
print(args.add_note, file=table, end=';')
p = util.get_param(str(args))
print(p, file=table, end=';')
long, short = util.get_command(str(args))
p = "python LanguageModel.py "
long = p + long
print(long, file=table, end='\n')
def disable_account(account_name=None):
if account_name is None:
account_name = util.get_param(2, None)
if account_name is None:
p("}}enInvalid User name - not disabling account!}}xx")
return False
try:
user_data = dict()
user_data['flags'] = UserAccounts.DISABLE_ACCOUNT_FLAGS
#win32netcon.UF_SCRIPT | win32netcon.UF_ACCOUNTDISABLE
win32net.NetUserSetInfo(None, account_name, 1008, user_data)
except Exception as ex:
p("}}rnError - Unable to disable account: " + str(account_name) + "}}xx\n" + \
str(ex))
return False
return True
def set_log_level():
# Set the log level parameter in the registry
param = util.get_param(2)
if param == "":
p("}}rnInvalid Log Level Specified}}xx")
return False
log_level = 3
try:
log_level = int(param)
except:
p("}}rnInvalid Log Level Specified}}xx")
return False
# Set the registry setting
RegistrySettings.set_reg_value(app="OPEService", value_name="log_level", value=log_level)
return True
def set_scan_nics_timer():
# Set how often to reload scan nics
param = util.get_param(2)
if param == "":
p("}}rnInvalid frequency Specified}}xx")
return False
frequency = 60
try:
frequency = int(param)
except:
p("}}rnInvalid frequency Specified}}xx")
return False
# Set the registry setting
RegistrySettings.set_reg_value(app="OPEService", value_name="scan_nics_timer", value=frequency)
return True
def set_default_permissions_timer():
# Set how often to reset permissions in the registry
param = util.get_param(2)
if param == "":
p("}}rnInvalid frequency Specified}}xx")
return False
frequency = 3600
try:
frequency = int(param)
except:
p("}}rnInvalid frequency Specified}}xx")
return False
# Set the registry setting
RegistrySettings.set_reg_value(app="OPEService", value_name="set_default_permissions_timer", value=frequency)
return True
def __init__(self, dataset, center_loader=None,
param=None, use_gpu=False):
self._dataset = dataset
self._center_loader = center_loader
init_proto_name, init_model_name = util.get_model(dataset, 'baseline')
proto_name, model_name = util.get_model(dataset, 'pose_ren')
self._fx, self._fy, self._ux, self._uy = util.get_param(dataset) if param is None else param
self._net = caffe.Net(proto_name, caffe.TEST, weights=model_name)
self._net_init = caffe.Net(init_proto_name, caffe.TEST, weights=init_model_name)
self._input_size = self._net.blobs['data'].shape[-1]
self._cube_size = 150
if use_gpu:
caffe.set_mode_gpu()
caffe.set_device(0)
else:
caffe.set_mode_cpu()
def set_screen_shot_timer():
# Set how often to reload scan nics
param = util.get_param(2)
if param == "":
p("}}rnInvalid frequency Specified}}xx")
return False
frequency = "30-300"
try:
frequency = str(param)
except:
# Not an int? Set it as a string
frequency = param
#p("}}rnInvalid frequency Specified}}xx")
#return False
# Set the registry setting
RegistrySettings.set_reg_value(app="OPEService", value_name="screen_shot_timer",
value=frequency)
return True
def apply_group_policy():
ret = True
if RegistrySettings.is_debug():
p("}}rbDEBUG MODE ON - Skipping apply group policy}}xx")
return True
gpo_name = util.get_param(2, "gpo")
# LGPO.exe is in the rc sub folder of the mgmt tool
app_folder = util.get_app_folder()
lgpo_path = os.path.join(app_folder, "rc")
cmd = "lgpo.exe /g " + gpo_name
returncode, output = ProcessManagement.run_cmd(cmd,
cwd=lgpo_path,
attempts=5,
require_return_code=0,
cmd_timeout=30)
if returncode == -2:
# Unable to restore gpo?
p("}}rnERROR - Unable to set group policy!}}xx\n" + output)
ret = False
return ret
# Force gpupdate
cmd = "%SystemRoot%\\system32\\gpupdate /force"
returncode, output = ProcessManagement.run_cmd(cmd,
attempts=5,
require_return_code=0,
cmd_timeout=30)
if returncode == -2:
# Error running command?
p("}}rnERROR - Unable to set force gpupdate!}}xx\n" + output)
ret = False
return ret
def set_default_groups_for_admin(account_name=None):
ret = True
# Make sure the student is in the proper groups
if account_name is None:
account_name = util.get_param(2, None)
if account_name is None:
p("}}rnInvalid User name - not adding default admin groups to account!}}xx"
)
return False
# Make sure students group exists
ret = UserAccounts.create_local_students_group()
if not UserAccounts.add_user_to_group(account_name, "Administrators"):
ret = False
if not UserAccounts.add_user_to_group(account_name, "Users"):
ret = False
# # home_dir = "%s\\%s" % (server_name, user_name)
#
return ret
def log_out_user(user_name=None):
if user_name is None:
user_name = util.get_param(2, None)
if user_name is None:
p("}}rn No User name provided - not logging out!}}xx")
return False
# Get list of current sessions
sessions = win32ts.WTSEnumerateSessions(None, 1, 0)
logged_off = False
for session in sessions:
active_session = session['SessionId']
station_name = session["WinStationName"]
# Get the user for this session
logged_in_user_name = win32ts.WTSQuerySessionInformation(
None, active_session, win32ts.WTSUserName)
#p("}}ynComparing: " + str(user_name) + "/" + logged_in_user_name)
if user_name == logged_in_user_name:
# Log this one out
p("}}gnLogging off " + str(user_name) +
" - typically takes 10-120 seconds...}}xx",
debug_level=4)
win32ts.WTSLogoffSession(None, active_session, True)
logged_off = True
if logged_off is not True:
p("}}ybUser not logged in - skipping log off! " + str(user_name) +
"}}xx",
debug_level=5)
else:
p("}}gnUser logged out! " + str(user_name) + "}}xx", debug_level=3)
return True
def save_log(mode="w"):
with open(log_path, mode) as outFile:
if mode == "a":
print("-----------resume the training ---------", file=outFile)
else:
print("-----------Language Model---------", file=outFile)
print("file name = ", CHECKPOINTS_LM + args.save_to, file=outFile)
print("", file=outFile)
long, short = util.get_command(str(args))
p = "python LanguageModel.py "
long = p + long
short = p + short
print(">>> command with full parameters", file=outFile)
print(long, file=outFile)
print("\n>>> command with short parameters", file=outFile)
print(short, file=outFile)
print("", file=outFile)
print(">>> ", args.add_note, file=outFile)
print("trainLosses ", trainLosses, file=outFile)
print("devLosses ", devLosses, file=outFile)
print("", file=outFile)
l = str(args)[10:].strip(")").split(",")
for i in l:
print(i, file=outFile)
print("", file=outFile)
print("train set: ", train_path, file=outFile)
print("dev set: ", dev_path, file=outFile)
print("", file=outFile)
print("config for later download : ", file=outFile)
p = util.get_param(str(args))
print(p, file=outFile)
print("", file=outFile)
print("save log file to ", args.save_to)
def user_email_delete():
#region data from param
#user_email_id = request.values.get('user_email_id')
user_email_id = get_param(request, name='user_email_id', required=True)
#endregion data from param
#get userEmail obj from user_email_id
from model.user_email import UserEmail
userEmail = UserEmail.query.get(user_email_id)
if not userEmail:
raise Exception('UserEmail not found user_email_id=%s' % user_email_id)
#do delete & store to db
db.session.delete(userEmail)
db.session.commit()
#output
d = {
'message': 'Deleted user user_id=%s' % user_email_id,
'data': userEmail.toDict(),
}
return jsonify(
d
) #return JSON in Flask ref. http://stackoverflow.com/a/13089975/248616
def list_system_nics():
NetworkDevices.init_device_list()
verbose = util.get_param(2, "").strip("-")
if verbose == "verbose":
verbose = "v"
# Get a list of nics currently in the system
nics_found = NetworkDevices.get_nics_w32()
for nic in nics_found:
(nic_name, nic_network_name, nic_ip_addresses, nic_connected,
nic_enabled, attribs, iface, nic_if_index) = nic
filtered_nic_ip_addresses = NetworkDevices.filter_local_link_ip_addresses(
nic_ip_addresses)
if nic_network_name != "None" and nic_network_name is not None:
nic_network_name = "}}wn(}}cn" + "{0:>10}".format(
nic_network_name) + "}}wn)"
else:
nic_network_name = "" # "}}wn(" + "{0:>10}".format("None") + ")"
if nic_ip_addresses is None:
nic_ip_addresses = []
if nic_connected is True:
nic_connected = "}}yb" + "Connected".ljust(12) + "}}xx"
else:
nic_connected = "}}yn" + "Disconnected".ljust(12) + "}}xx"
# 1-Up, 2-Down, 3-Testing
if nic_enabled == 0:
nic_enabled = "}}yb" + "Enabled".rjust(9) + "}}xx"
elif nic_enabled == 22:
nic_enabled = "}}yn" + "Disabled".rjust(9) + "}}xx"
else:
nic_enabled = "}}rn" + "Unplugged".rjust(9) + "}}xx"
# Use nic description to check approved status - no #2 etc... at the end
nic_description = iface.Description
# Not approved - show as X
approved_status = "}}rbX}}xx"
# Returns T/F for (matched_name, matched_ip)
r = NetworkDevices.is_nic_approved(nic_description,
filtered_nic_ip_addresses)
num_ips = len(filtered_nic_ip_addresses)
if r[0] is True and r[1] is True:
# Approved and good IP - show as good
approved_status = "}}gb+}}xx"
elif r[0] is True and r[1] is False and num_ips == 0:
# Approved but no IP
approved_status = "}}yb?}}xx"
elif r[0] is True and r[1] is False and num_ips > 0:
approved_status = "}}rb!}}xx"
ip_address_str = ""
if len(nic_ip_addresses) > 0:
ip_address_str = "\n\t}}cn" + str(nic_ip_addresses) + "}}xx"
p(approved_status + \
"}}gn" + " {0:>2}".format(nic_if_index) + \
" }}wn" + "{0:43}".format(nic_name) + \
" " + nic_enabled + \
"/" + nic_connected + \
" " + nic_network_name + \
" " + ip_address_str + \
"}}xx")
if verbose == "v":
p("}}cn-- " + str(attribs) + "}}xx")
p("\n}}gb+ }}xxApproved/Connecte, }}yb? }}xxApproved/Not Connected, }}rbX }}xxNot Approved}}xx, }}rb! }}xxApproved/Invalid IP}}xx"
)
p("}}rbNOTE}}xx - Approved Nics w/out IPs (or with 169.254/fe80::) will be enabled so they can use DHCP}}xx"
)
return True
},
"version": {
"function": show_version,
"help": "Display the version for the LMS software",
"require_admin": False
},
}
if __name__ == "__main__":
# returns (is in admins, is uac, curr_user_name)
is_admin = ensure_admin()
# Parse Arguments
cmd = util.get_param(1).lower()
if cmd not in valid_commands:
# Unknown Command??
p("}}rnInvalid Command! - " + str(cmd) + "}}xx", log_level=1)
# Only show commands if UAC active
if is_admin[1]:
commands = sorted(valid_commands.keys())
p("}}yn Valid Commands: " + str(commands) + "}}xx")
p("}}ybFor help - type mgmt.exe help (command)}}xx")
sys.exit(1)
# Run the function associated w the command
cmd_parts = valid_commands[cmd]
if cmd_parts is None:
img /= 1160
img *= 255
points = centers[idx]
img = util.draw_pose(dataset, img, lbl, 3, (255, 0, 0), points)
cv2.imwrite(
root_dir + 'samples/depth/' + phase + '_' + str(idx) + '.png', img)
##############################################################################################
############################# Normalize Labels #################################
### Make sure centers are calculated using get_centers.py before running this
### Also make sure you are working with original non normalized 3D joint coordinates not the normalized ones.
### After running this segment you should rename the original and give this new file the original's name so that it becomes the labels file used moving forward
################################################################################
lbls = util.load_labels(dataset, phase) # load original test/train data
centers = util.load_centers(dataset, phase).astype(float)
fx, fy, ux, uy = util.get_param(dataset)
out_file = root_dir + 'labels/fpad_' + phase + '_label_nm.txt'
lbls = np.asarray([s.split() for s in lbls], dtype=np.float32)
for lid, lbl in enumerate(lbls):
joints = np.asarray(np.reshape(lbl, (21, 3)), dtype=np.float32)
joints, skel_camcoords = util.world2pixel(joints, dataset)
lbls[lid] = np.reshape(joints, (63))
lbls = np.reshape(lbls, (-1, 63))
x = util.normalize_pose(dataset, lbls, centers, 150, fx, fy)
util.save_results(x, out_file)
################################################################################
def test():
global model
count_test_samples = 0
correct = 0
falsePositives = 0
falseNegatives = 0
count_real_word = 0
count_predict_word = 0
word_label = []
word_pred = []
word_correct = 0
precision = 0
model.rnn.train(False)
print("testing......")
test_data_CL = load_data_tokenizer(
test_path,
len_chunk=args.len_lines_per_chunk,
doShuffling=False,
)
test_chars = create_tensor_classifier(test_data_CL)
while True:
try:
numeric = next(test_chars)
except StopIteration:
break
loss, numberOfCharacters, log_probs_cal, input_tensor_cal, target_tensor_cal = model.forward_cl(
numeric, train=False)
count_test_samples += args.batchSize
tag_score = log_probs_cal
flat_input = input_tensor_cal.transpose(1, 0)
flat_label = target_tensor_cal.transpose(1, 0)
tag_score = tag_score.transpose(0, 1)
val, argmax = tag_score.max(dim=2)
for b in range(args.batchSize):
chars = [
itos[element.item()] if element.item() != 0 else "-"
for element in flat_input[b]
]
pred_seq = chars.copy()
label = flat_label[b]
pred = argmax[b]
for i in range(args.sequence_length):
if label[i] == 1:
chars[i] = " " + chars[i]
if pred[i] == 1:
pred_seq[i] = " " + pred_seq[i]
if pred[i] == label[i]:
correct += 1
if pred[i] != label[i] and pred[i] == 0:
falseNegatives += 1
if pred[i] != label[i] and pred[i] == 1:
falsePositives += 1
# word level evaluation
word_label.append(label[i])
word_pred.append(pred[i])
if pred[i] == 1:
count_predict_word += 1
if label[i] == 1:
if word_label == word_pred:
word_correct += 1
word_pred = []
word_label = []
count_real_word += 1
if printAllPrediction and "".join(chars) != "".join(pred_seq):
print("".join(chars))
print("".join(pred_seq))
if printPrediction and "".join(chars) != "".join(pred_seq):
print("".join(chars))
print("".join(pred_seq))
print("train losses ", trainLosses)
print("dev losses ", devLosses)
print()
print("============Evaluation===========")
print()
if correct == 0:
print("correct predicted boundaries ", correct)
print("correct predicted words ", word_correct)
else:
precision = correct / (correct + falsePositives)
recall = correct / (correct + falseNegatives)
f1 = 2 * (precision * recall) / (precision + recall)
print("Boundary Measure")
print("False Negatives ", falseNegatives)
print("False Positives ", falsePositives)
print("Correctly predicted boundaries", correct)
print("Precision", round(precision * 100, 2), "Recall",
round(recall * 100, 2), "F1", round(f1 * 100, 2))
print()
print("Word Measure")
word_precision = word_correct / (count_predict_word)
word_recall = word_correct / (count_real_word)
word_f1 = 2 * (word_precision * word_recall) / (word_precision +
word_recall)
print("Correctly predicted words", word_correct)
print("Number of predicted words", count_predict_word)
print("Number of real words", count_real_word)
print("Precision", round(word_precision * 100, 2), "Recall",
round(word_recall * 100, 2), "F1", round(word_f1 * 100, 2))
print()
print("sample result")
print("label : " + "".join(chars))
print("predict: " + "".join(pred_seq))
with open(log_path, "a+") as outFile:
print("falseNegatives ", falseNegatives, file=outFile)
print("falsePositives ", falsePositives, file=outFile)
print("correct ", correct, file=outFile)
print("Boundary measures: ", file=outFile)
print("Precision",
precision,
"Recall",
recall,
"F1",
2 * (precision * recall) / (precision + recall),
file=outFile)
print("\nWord measures",
"Precision",
word_precision,
"Recall",
word_recall,
"F1",
2 * (word_precision * word_recall) /
(word_precision + word_recall),
file=outFile)
print("word_correct :", word_correct, file=outFile)
print("count_predict_word: ", count_predict_word, file=outFile)
print("count_real_word: ", count_real_word, file=outFile)
print("", file=outFile)
print("".join(chars), file=outFile)
print("".join(pred_seq), file=outFile)
print(total_time, file=outFile)
csv_path = os.path.join(CHECKPOINTS_TOKENIZER, "tokenizer_result.csv")
with open(csv_path, "a+") as table:
print()
print("..........................")
print("-", file=table, end=';')
if str(args.load_from)[:2] == "LM":
print("LM", file=table, end=';')
else:
print("-", file=table, end=';')
print(args.save_to, file=table, end=';')
print(args.dataset, file=table, end=';')
print("boundary", file=table, end=';')
precision = round(precision * 100, 2)
recall = round(recall * 100, 2)
f1 = round(f1 * 100, 2)
print(precision, file=table, end=';')
print(recall, file=table, end=';')
print(f1, file=table, end=';')
print("word", file=table, end=';')
word_precision = round(word_precision * 100, 2)
word_recall = round(word_recall * 100, 2)
word_f1 = round(word_f1 * 100, 2)
print(word_precision, file=table, end=';')
print(word_recall, file=table, end=';')
print(word_f1, file=table, end=';')
print(num_epoch + 1, file=table, end=';')
print(total_time, file=table, end=';')
print("trainLosses ", trainLosses, file=table, end=';')
print("devLosses ", devLosses, file=table, end=';')
print(args.add_note, file=table, end=';')
p = util.get_param(str(args))
print(p, file=table, end=';')
long, short = util.get_command(str(args))
p = "python Tokenizer.py "
long = p + long
print(long, file=table, end='\n')
def compute_layer_stats(layer):
refreeze = False
if hasattr(layer, 'frozen') and layer.frozen:
u.unfreeze(layer)
refreeze = True
s = AttrDefault(str, {})
n = args.stats_batch_size
param = u.get_param(layer)
_d = len(param.flatten()) # dimensionality of parameters
layer_idx = model.layers.index(layer)
# TODO: get layer type, include it in name
assert layer_idx >= 0
assert stats_data.shape[0] == n
def backprop_loss():
model.zero_grad()
output = model(
stats_data) # use last saved data batch for backprop
loss = compute_loss(output, stats_targets)
loss.backward()
return loss, output
def backprop_output():
model.zero_grad()
output = model(stats_data)
output.backward(gradient=torch.ones_like(output))
return output
# per-example gradients, n, d
_loss, _output = backprop_loss()
At = layer.data_input
Bt = layer.grad_output * n
G = u.khatri_rao_t(At, Bt)
g = G.sum(dim=0, keepdim=True) / n
u.check_close(g, u.vec(param.grad).t())
s.diversity = torch.norm(G, "fro")**2 / g.flatten().norm()**2
s.grad_fro = g.flatten().norm()
s.param_fro = param.data.flatten().norm()
pos_activations = torch.sum(layer.data_output > 0)
neg_activations = torch.sum(layer.data_output <= 0)
s.a_sparsity = neg_activations.float() / (
pos_activations + neg_activations) # 1 sparsity means all 0's
activation_size = len(layer.data_output.flatten())
s.a_magnitude = torch.sum(layer.data_output) / activation_size
_output = backprop_output()
B2t = layer.grad_output
J = u.khatri_rao_t(At, B2t) # batch output Jacobian
H = J.t() @ J / n
s.hessian_l2 = u.l2_norm(H)
s.jacobian_l2 = u.l2_norm(J)
J1 = J.sum(dim=0) / n # single output Jacobian
s.J1_l2 = J1.norm()
# newton decrement
def loss_direction(direction, eps):
"""loss improvement if we take step eps in direction dir"""
return u.to_python_scalar(eps * (direction @ g.t()) - 0.5 *
eps**2 * direction @ H @ direction.t())
s.regret_newton = u.to_python_scalar(g @ u.pinv(H) @ g.t() / 2)
# TODO: gradient diversity is stuck at 1
# TODO: newton/gradient angle
# TODO: newton step magnitude
s.grad_curvature = u.to_python_scalar(
g @ H @ g.t()) # curvature in direction of g
s.step_openai = u.to_python_scalar(
s.grad_fro**2 / s.grad_curvature) if s.grad_curvature else 999
s.regret_gradient = loss_direction(g, s.step_openai)
if refreeze:
u.freeze(layer)
return s
def approve_nic():
# Get the params for the nic
nic_name = util.get_param(2)
nic_network = util.get_param(3)
if nic_name == "" or nic_network == "":
p("}}rbError - Invalid paramaters! try mgmt.exe help approve_nic for more information}}xx"
)
return False
nic_id = None
try:
nic_id = int(nic_name)
except:
# Ok if this fails - trying to see if an ID was passed instead of a name
pass
if not nic_id is None:
# Lookup the nic by ID
iface = NetworkDevices.get_nic_by_interface_index(nic_id)
if iface is None:
# Unable to find an interface by this ID
p("}}rbInvalid Interface ID! " + str(nic_id) + "}}xx")
return False
else:
#nic_name = iface.Name
# NOTE - Description will give us the driver name w/out the #2 after it
nic_name = iface.Description
# Strip off #? at the end of the name
t_nic = nic_name
removed_suffix = ""
for i in range(1, 30):
suffix = " #" + str(i)
if suffix in t_nic:
t_nic = t_nic.replace(suffix, "")
removed_suffix = suffix
if removed_suffix != "":
p("}}ybNOTE - Stripped off " + removed_suffix +
" from nic name\n - All instances of this nic approved with this network}}xx"
)
nic_name = t_nic
p("}}gnApproving " + nic_name + " on netowrk " + nic_network,
log_level=1)
# Get the list of approved nics
# Load the value from the registry
try:
approved_nics_json = RegistrySettings.get_reg_value(
app="OPEService", value_name="approved_nics", default="[]")
nic_list = json.loads(approved_nics_json)
# Add this nic to the list
nic_list.append((nic_name, nic_network))
# Write this back to the registry
approved_nics_json = json.dumps(nic_list)
RegistrySettings.set_reg_value(app="OPEService",
value_name="approved_nics",
value=approved_nics_json)
except Exception as ex:
p("}}rbUnable to write approved nics to the registry!}}xx",
log_level=1)
return False
# Force a reload of the device list
NetworkDevices.init_device_list(refresh=True)
return True
Author: Hengkai Guo, Xinghao Chen
"""
import numpy as np
import matplotlib.pyplot as plt
import util
show_validate_figure = 0
# load hand poses
labels = np.loadtxt('../labels/msra_test_label.txt')
labels = np.reshape(labels, (-1, 21, 3))
# first convert uvd to xyz
params = util.get_param('msra')
labels = util.pixel2world(labels, *params)
# wrist, mcp for index, middle, ring, little, thumb
labels = labels[:, [0, 1, 5, 9, 13, 17], :]
'''
The palm coordinate frame has its origin at wrist, positive Y axis
pointing to the middle finger root and positive Z axis pointing
outwards of the palm plane.
Sun et al. "Cascaded Hand Pose Regression", CVPR 2015
'''
# y axis
y = labels[:, 2, :] - labels[:, 0, :]
# z axis
def start_upgrade_process(branch=None, force_upgrade=None):
ret = True
if not CredentialProcess.is_time_to_upgrade():
p("}}gnNot time to check for upgrades yet, skipping...}}xx", log_level=3)
return True
RegistrySettings.set_reg_value(value_name="last_upgrade_time", value=time.time())
curr_branch = branch
if curr_branch is None:
# See if a parameter was provided
curr_branch = util.get_param(2, None)
# Force upgrade - even if versions match
if force_upgrade is None:
force_upgrade = util.get_param(3, "")
if force_upgrade.lower() != "-f":
force_upgrade = False
else:
force_upgrade = True
# If branch is still empty, get it from the registry
if curr_branch is None:
curr_branch = RegistrySettings.get_reg_value(value_name="install_branch", default="master")
# Start by grabbing any new stuff from the git server
ret = ProcessManagement.git_pull_branch(curr_branch)
if ret is False:
# Not critical if this fails - apply whatever is present if it is
# a different version number
# return False
p("}}ybWARNING - Unable to pull updates for git server!}}xx")
pass
# Save the current branch for next time
RegistrySettings.set_reg_value(value_name="install_branch", value=curr_branch)
# Check the mgmt.version files to see if we have a new version
ope_laptop_binaries_path = os.path.expandvars("%programdata%\\ope\\tmp\\ope_laptop_binaries")
# Get the path to the mgmt.version file
git_version_path = os.path.join(ope_laptop_binaries_path, "Services", "mgmt", "mgmt.version")
# Do we have a new version?
curr_version = CredentialProcess.get_mgmt_version()
git_version = CredentialProcess.get_mgmt_version(git_version_path)
if git_version == "NO VERSION":
# No version file found
p("}}ynNo version file found in git repo, skipping upgrade!}}xx")
return False
if not CredentialProcess.is_version_newer(curr_version, git_version) and force_upgrade is not True:
# Same version - no upgrade needed
p("}}gnOPE Software up to date: " + str(git_version) + " not newer than " + str(curr_version) + " - (not upgrading)}}xx")
return True
# Version is different, prep for update
forced = ""
if force_upgrade:
forced = "}}yb(upgrade forced)}}gn"
p("}}gnFound new version " + forced + " - starting upgrade process: " + \
curr_version + " --> " + git_version + "}}xx")
# Lock user accounts
if not UserAccounts.disable_student_accounts():
p("}}rbERROR - Unable to disable student accounts prior to upgrade!}}xx")
return False
# Make sure students are logged out
if not UserAccounts.log_out_all_students():
p("}}rbERROR - Unable to log out student accounts prior to upgrade!}}xx")
return False
p("}}ynLaunching OPE Software Update process...}}xx")
# run the upgrade_ope.cmd from the TMP rc folder!!!
bat_path = os.path.join(ope_laptop_binaries_path, "Services\\mgmt\\rc\\upgrade_ope.cmd")
# Add the redirect so we end up with a log file
if not ProcessManagement.run_detatched_cmd(bat_path + " >> %programdata%\\ope\\tmp\\log\\upgrade.log 2>&1"):
p("}}rbERROR - Unable to start upgrade process!}}xx")
return False
# Make sure to exit this app??
#sys.exit(0)
return ret
