def show_help(): global LOGGER, valid_commands # Find the help key for this command cmd = util.get_param(1).lower() param1 = util.get_param(2).lower() if cmd == "" or param1 == "": # Missing required parameters! p("}}rnMissing Required Parameters! " + cmd + " - " + param1 + "}}xx", log_level=1) return False if not param1 in valid_commands: p("}}rnInvalid Command! " + param1 + "}}xx", log_level=1) commands = list(valid_commands.keys()) p("}}yn Valid Commands: " + str(commands) + "}}xx") p("}}ybFor help - type mgmt.exe help (command)}}xx") return False cmd_parts = valid_commands[param1] if cmd_parts is None: p("}}rnInvalid Command - not configured! " + param1 + "}}xx", log_level=1) return False help_msg = cmd_parts["help"] if help_msg is None: p("}}rnNo Help Provided! " + param1 + "}}xx", log_level=1) return False p("}}yb" + help_msg + "}}xx") return True
def remove_nic(): # Get the params for the nic nic_name = util.get_param(2) nic_network = util.get_param(3) if nic_name == "" or nic_network == "": p("}}rbError - Invalid paramaters!}}xx") return False # Get the list of approved nics # Load the value from the registry try: approved_nics_json = RegistrySettings.get_reg_value( app="OPEService", value_name="approved_nics", default="[]") nic_list = json.loads(approved_nics_json) new_nic_list = [] # Loop through old list and add any that don't match remove parameters removed = False for item in nic_list: #p("checking " + item[0] + " -> " + item[1]) if item[0] == nic_name and item[1] == nic_network: #p("FOUND!") removed = True else: # Not the one to remove, put it in the new_nic_list new_nic_list.append(item) if removed: # Write this back to the registry approved_nics_json = json.dumps(new_nic_list) RegistrySettings.set_reg_value(app="OPEService", value_name="approved_nics", value=approved_nics_json) p("}}gnRemoved " + nic_name + " on netowrk " + nic_network + "}}xx", log_level=1) else: p("}}rnNOT FOUND - NOT Removed " + nic_name + " on netowrk " + nic_network + " (Can't remove system pre-approved nic entries)}}xx") except Exception as ex: p("}}rbUnable to write approved nics to the registry!}}xx\n" + str(ex), log_level=1) return False # Force device list refresh NetworkDevices.init_device_list(refresh=True) return True
def __init__(self, task): # Task (environment) information self.task = task # should contain observation_space and action_space self.state_size = np.prod(3) self.state_range = self.task.observation_space.high[ 0:3] - self.task.observation_space.low[0:3] self.action_size = np.prod(3) self.action_range = self.task.action_space.high[ 0:3] - self.task.action_space.low[0:3] # Policy parameters self.w = np.random.normal( size=( 3, 3 ), # weights for simple linear policy: state_space x action_space self.state_size scale=(self.action_range / (2 * self.state_size)).reshape( 1, -1)) # start producing actions in a decent range # Score tracker and learning parameters self.best_w = None self.best_score = -np.inf self.noise_scale = 0.1 self.episode_num = 1 self.stats_filename = os.path.join( util.get_param('out'), "stats_{}.csv".format(util.get_timestamp())) # path to CSV file self.stats_columns = ['episode', 'total_reward'] # specify columns to save # Episode variables self.reset_episode_vars()
def export_group_policy(): ret = True gpo_name = util.get_param(2, "exported_gpo") # LGPO.exe is in the rc sub folder of the mgmt tool app_folder = util.get_app_folder() lgpo_path = os.path.join(app_folder, "rc") gpo_path = os.path.join(lgpo_path, gpo_name) # NOTE - lgpo needs full path to gpo export directory cmd = "lgpo.exe /b \"" + gpo_path + "\"" # Remove the folder if it exists p("Removing old files: " + gpo_path) shutil.rmtree(gpo_path, ignore_errors=True) # Folder has to exist os.makedirs(gpo_path, exist_ok=True) returncode, output = ProcessManagement.run_cmd(cmd, cwd=lgpo_path, attempts=5, require_return_code=0, cmd_timeout=15) if returncode == -2: # Unable to restore gpo? p("}}rnERROR - Unable to dump group policy!}}xx\n" + output) ret = False return ret p(output) return ret
def delete_user(user_name=None): if user_name is None: user_name = util.get_param(2, None) if user_name is None: p("}}enInvalid User name - not removing account!}}xx") return False curr_user = None try: curr_user = accounts.principal(user_name) except Exception as ex: p("}}rbInvalid User Account - Not deleting!}}xx", debug_level=1) return False # Remove the profile first ret = UserAccounts.remove_account_profile(user_name) # Remove the local user try: curr_user.delete() except Exception as ex: p("}}rbError - Unable to remove account: " + str(user_name) + "}}xx\n" + str(ex)) return False return True
def apply_firewall_policy(): ret = True if RegistrySettings.is_debug(): p("}}rbDEBUG MODE ON - Skipping apply firewall policy}}xx") return True policy_file_name = util.get_param(2, "firewall_config.wfw") # Should be in RC sub folder under the app app_folder = util.get_app_folder() rc_path = os.path.join(app_folder, "rc") policy_file_path = os.path.join(rc_path, policy_file_name) # netsh advfirewall import "%~dp0rc\firewall_config.wfw" 2>NUL 1<NUL cmd = "%SystemRoot%\\system32\\netsh advfirewall import \"" + policy_file_path + "\"" returncode, output = ProcessManagement.run_cmd(cmd, attempts=5, require_return_code=0, cmd_timeout=15) if returncode == -2: # Error running command? p("}}rbError - Unable to reset firewall back to defaults?}}xx\n" + output) ret = False return ret
def __init__(self, dataset, model, center_loader=None, param=None): self._dataset = dataset self._center_loader = center_loader proto_name, model_name = util.get_model(dataset, model) self._fx, self._fy, self._ux, self._uy = util.get_param(dataset) if param is None else param self._net = caffe.Net(proto_name, caffe.TEST, weights=model_name) self._input_size = self._net.blobs['data'].shape[-1] self._cube_size = 150
def save_log(mode="w"): with open(log_path, mode) as outFile: if mode == "a": print("----------resume training/early stopping ---------", file=outFile) else: print("-----------Tokenizer---------", file=outFile) # commands for later training long, short = util.get_command(str(args)) p = "python Tokenizer.py " long = p + long short = p + short print(">>> command with full parameters", file=outFile) print(long, file=outFile) print("\n>>> command with short parameters", file=outFile) print(short, file=outFile) print("", file=outFile) # model parameters and losses print("file name = ", args.save_to, file=outFile) print("", file=outFile) print(">>> ", args.add_note, file=outFile) print("bi_lstm ", bi_lstm, file=outFile) print("train classifier for " + str(args.epoch) + " epoch", file=outFile) print("trainLosses ", trainLosses, file=outFile) print("devLosses ", devLosses, file=outFile) print("", file=outFile) print("count train sample ", count_train_samples, file=outFile) print("count dev sample ", count_dev_samples, file=outFile) # model parameters print("", file=outFile) l = str(args)[10:].strip(")").split(",") for i in l: print(i, file=outFile) print("", file=outFile) # data set print("train set: ", train_path, file=outFile) print("dev set: ", dev_path, file=outFile) print("test set: ", test_path, file=outFile) print("", file=outFile) # number of samples print("count train sample ", count_train_samples, file=outFile) print("count dev sample ", count_dev_samples, file=outFile) # parameter flags (used when later import this model to another model) print("config for later download : ", file=outFile) p = util.get_param(str(args)) print(p, file=outFile) print("", file=outFile)
def lock_screen_for_user(user_name=None): # Find the user in question and lock the workstation if user_name is None: user_name = util.get_param(2, None) if user_name is None: # Lock for the current user if no name return UserAccounts.lock_screen_for_current_user() p("}}ybLocking screen for other users - Not Implemented Yet!}}xx") # TODO - lock_workstation # Lookup the user specified and run this under their account return False
def ping_smc(smc_url=None): # See if we can bounce off the SMC server and get a response if smc_url is None: # Try and get from command line smc_url = util.get_param(2, None) if smc_url is None: # Nothing on command line? Get from registry smc_url = RegistrySettings.get_reg_value(value_name="smc_url", default="https://smc.ed") force = util.get_param(3, "") if force == "-f": force = True else: force = False if not force and not CredentialProcess.is_time_to_ping_smc(): #p("}}gnNot time to ping smc, skipping...}}xx", log_level=4) return True RegistrySettings.set_reg_value(value_name="last_smc_ping_time", value=time.time()) if not RestClient.ping_smc(smc_url): p("}}mnNot able to ping SMC " + smc_url + "}}xx", log_level=4) # Ok to return true - we just don't do more maintenance return True # We are connected, do maintenance # Check if time to sync time SystemTime.sync_time_w_ntp() # Check if time to sync stuff (lms app, folders, logs)... # TODO # Check if time to auto upgrade CredentialProcess.start_upgrade_process() return True
def save_csv(f="LM_log.csv"): csv_path = os.path.join(CHECKPOINTS_LM, f) with open(csv_path, "a+") as table: print(args.save_to, file=table, end=';') print(args.dataset, file=table, end=';') print(num_epoch + 1, file=table, end=';') print("trainLosses ", trainLosses, file=table, end=';') print("devLosses ", devLosses, file=table, end=';') print(args.add_note, file=table, end=';') p = util.get_param(str(args)) print(p, file=table, end=';') long, short = util.get_command(str(args)) p = "python LanguageModel.py " long = p + long print(long, file=table, end='\n')
def disable_account(account_name=None): if account_name is None: account_name = util.get_param(2, None) if account_name is None: p("}}enInvalid User name - not disabling account!}}xx") return False try: user_data = dict() user_data['flags'] = UserAccounts.DISABLE_ACCOUNT_FLAGS #win32netcon.UF_SCRIPT | win32netcon.UF_ACCOUNTDISABLE win32net.NetUserSetInfo(None, account_name, 1008, user_data) except Exception as ex: p("}}rnError - Unable to disable account: " + str(account_name) + "}}xx\n" + \ str(ex)) return False return True
def set_log_level(): # Set the log level parameter in the registry param = util.get_param(2) if param == "": p("}}rnInvalid Log Level Specified}}xx") return False log_level = 3 try: log_level = int(param) except: p("}}rnInvalid Log Level Specified}}xx") return False # Set the registry setting RegistrySettings.set_reg_value(app="OPEService", value_name="log_level", value=log_level) return True
def set_scan_nics_timer(): # Set how often to reload scan nics param = util.get_param(2) if param == "": p("}}rnInvalid frequency Specified}}xx") return False frequency = 60 try: frequency = int(param) except: p("}}rnInvalid frequency Specified}}xx") return False # Set the registry setting RegistrySettings.set_reg_value(app="OPEService", value_name="scan_nics_timer", value=frequency) return True
def set_default_permissions_timer(): # Set how often to reset permissions in the registry param = util.get_param(2) if param == "": p("}}rnInvalid frequency Specified}}xx") return False frequency = 3600 try: frequency = int(param) except: p("}}rnInvalid frequency Specified}}xx") return False # Set the registry setting RegistrySettings.set_reg_value(app="OPEService", value_name="set_default_permissions_timer", value=frequency) return True
def __init__(self, dataset, center_loader=None, param=None, use_gpu=False): self._dataset = dataset self._center_loader = center_loader init_proto_name, init_model_name = util.get_model(dataset, 'baseline') proto_name, model_name = util.get_model(dataset, 'pose_ren') self._fx, self._fy, self._ux, self._uy = util.get_param(dataset) if param is None else param self._net = caffe.Net(proto_name, caffe.TEST, weights=model_name) self._net_init = caffe.Net(init_proto_name, caffe.TEST, weights=init_model_name) self._input_size = self._net.blobs['data'].shape[-1] self._cube_size = 150 if use_gpu: caffe.set_mode_gpu() caffe.set_device(0) else: caffe.set_mode_cpu()
def set_screen_shot_timer(): # Set how often to reload scan nics param = util.get_param(2) if param == "": p("}}rnInvalid frequency Specified}}xx") return False frequency = "30-300" try: frequency = str(param) except: # Not an int? Set it as a string frequency = param #p("}}rnInvalid frequency Specified}}xx") #return False # Set the registry setting RegistrySettings.set_reg_value(app="OPEService", value_name="screen_shot_timer", value=frequency) return True
def apply_group_policy(): ret = True if RegistrySettings.is_debug(): p("}}rbDEBUG MODE ON - Skipping apply group policy}}xx") return True gpo_name = util.get_param(2, "gpo") # LGPO.exe is in the rc sub folder of the mgmt tool app_folder = util.get_app_folder() lgpo_path = os.path.join(app_folder, "rc") cmd = "lgpo.exe /g " + gpo_name returncode, output = ProcessManagement.run_cmd(cmd, cwd=lgpo_path, attempts=5, require_return_code=0, cmd_timeout=30) if returncode == -2: # Unable to restore gpo? p("}}rnERROR - Unable to set group policy!}}xx\n" + output) ret = False return ret # Force gpupdate cmd = "%SystemRoot%\\system32\\gpupdate /force" returncode, output = ProcessManagement.run_cmd(cmd, attempts=5, require_return_code=0, cmd_timeout=30) if returncode == -2: # Error running command? p("}}rnERROR - Unable to set force gpupdate!}}xx\n" + output) ret = False return ret
def set_default_groups_for_admin(account_name=None): ret = True # Make sure the student is in the proper groups if account_name is None: account_name = util.get_param(2, None) if account_name is None: p("}}rnInvalid User name - not adding default admin groups to account!}}xx" ) return False # Make sure students group exists ret = UserAccounts.create_local_students_group() if not UserAccounts.add_user_to_group(account_name, "Administrators"): ret = False if not UserAccounts.add_user_to_group(account_name, "Users"): ret = False # # home_dir = "%s\\%s" % (server_name, user_name) # return ret
def log_out_user(user_name=None): if user_name is None: user_name = util.get_param(2, None) if user_name is None: p("}}rn No User name provided - not logging out!}}xx") return False # Get list of current sessions sessions = win32ts.WTSEnumerateSessions(None, 1, 0) logged_off = False for session in sessions: active_session = session['SessionId'] station_name = session["WinStationName"] # Get the user for this session logged_in_user_name = win32ts.WTSQuerySessionInformation( None, active_session, win32ts.WTSUserName) #p("}}ynComparing: " + str(user_name) + "/" + logged_in_user_name) if user_name == logged_in_user_name: # Log this one out p("}}gnLogging off " + str(user_name) + " - typically takes 10-120 seconds...}}xx", debug_level=4) win32ts.WTSLogoffSession(None, active_session, True) logged_off = True if logged_off is not True: p("}}ybUser not logged in - skipping log off! " + str(user_name) + "}}xx", debug_level=5) else: p("}}gnUser logged out! " + str(user_name) + "}}xx", debug_level=3) return True
def save_log(mode="w"): with open(log_path, mode) as outFile: if mode == "a": print("-----------resume the training ---------", file=outFile) else: print("-----------Language Model---------", file=outFile) print("file name = ", CHECKPOINTS_LM + args.save_to, file=outFile) print("", file=outFile) long, short = util.get_command(str(args)) p = "python LanguageModel.py " long = p + long short = p + short print(">>> command with full parameters", file=outFile) print(long, file=outFile) print("\n>>> command with short parameters", file=outFile) print(short, file=outFile) print("", file=outFile) print(">>> ", args.add_note, file=outFile) print("trainLosses ", trainLosses, file=outFile) print("devLosses ", devLosses, file=outFile) print("", file=outFile) l = str(args)[10:].strip(")").split(",") for i in l: print(i, file=outFile) print("", file=outFile) print("train set: ", train_path, file=outFile) print("dev set: ", dev_path, file=outFile) print("", file=outFile) print("config for later download : ", file=outFile) p = util.get_param(str(args)) print(p, file=outFile) print("", file=outFile) print("save log file to ", args.save_to)
def user_email_delete(): #region data from param #user_email_id = request.values.get('user_email_id') user_email_id = get_param(request, name='user_email_id', required=True) #endregion data from param #get userEmail obj from user_email_id from model.user_email import UserEmail userEmail = UserEmail.query.get(user_email_id) if not userEmail: raise Exception('UserEmail not found user_email_id=%s' % user_email_id) #do delete & store to db db.session.delete(userEmail) db.session.commit() #output d = { 'message': 'Deleted user user_id=%s' % user_email_id, 'data': userEmail.toDict(), } return jsonify( d ) #return JSON in Flask ref. http://stackoverflow.com/a/13089975/248616
def list_system_nics(): NetworkDevices.init_device_list() verbose = util.get_param(2, "").strip("-") if verbose == "verbose": verbose = "v" # Get a list of nics currently in the system nics_found = NetworkDevices.get_nics_w32() for nic in nics_found: (nic_name, nic_network_name, nic_ip_addresses, nic_connected, nic_enabled, attribs, iface, nic_if_index) = nic filtered_nic_ip_addresses = NetworkDevices.filter_local_link_ip_addresses( nic_ip_addresses) if nic_network_name != "None" and nic_network_name is not None: nic_network_name = "}}wn(}}cn" + "{0:>10}".format( nic_network_name) + "}}wn)" else: nic_network_name = "" # "}}wn(" + "{0:>10}".format("None") + ")" if nic_ip_addresses is None: nic_ip_addresses = [] if nic_connected is True: nic_connected = "}}yb" + "Connected".ljust(12) + "}}xx" else: nic_connected = "}}yn" + "Disconnected".ljust(12) + "}}xx" # 1-Up, 2-Down, 3-Testing if nic_enabled == 0: nic_enabled = "}}yb" + "Enabled".rjust(9) + "}}xx" elif nic_enabled == 22: nic_enabled = "}}yn" + "Disabled".rjust(9) + "}}xx" else: nic_enabled = "}}rn" + "Unplugged".rjust(9) + "}}xx" # Use nic description to check approved status - no #2 etc... at the end nic_description = iface.Description # Not approved - show as X approved_status = "}}rbX}}xx" # Returns T/F for (matched_name, matched_ip) r = NetworkDevices.is_nic_approved(nic_description, filtered_nic_ip_addresses) num_ips = len(filtered_nic_ip_addresses) if r[0] is True and r[1] is True: # Approved and good IP - show as good approved_status = "}}gb+}}xx" elif r[0] is True and r[1] is False and num_ips == 0: # Approved but no IP approved_status = "}}yb?}}xx" elif r[0] is True and r[1] is False and num_ips > 0: approved_status = "}}rb!}}xx" ip_address_str = "" if len(nic_ip_addresses) > 0: ip_address_str = "\n\t}}cn" + str(nic_ip_addresses) + "}}xx" p(approved_status + \ "}}gn" + " {0:>2}".format(nic_if_index) + \ " }}wn" + "{0:43}".format(nic_name) + \ " " + nic_enabled + \ "/" + nic_connected + \ " " + nic_network_name + \ " " + ip_address_str + \ "}}xx") if verbose == "v": p("}}cn-- " + str(attribs) + "}}xx") p("\n}}gb+ }}xxApproved/Connecte, }}yb? }}xxApproved/Not Connected, }}rbX }}xxNot Approved}}xx, }}rb! }}xxApproved/Invalid IP}}xx" ) p("}}rbNOTE}}xx - Approved Nics w/out IPs (or with 169.254/fe80::) will be enabled so they can use DHCP}}xx" ) return True
}, "version": { "function": show_version, "help": "Display the version for the LMS software", "require_admin": False }, } if __name__ == "__main__": # returns (is in admins, is uac, curr_user_name) is_admin = ensure_admin() # Parse Arguments cmd = util.get_param(1).lower() if cmd not in valid_commands: # Unknown Command?? p("}}rnInvalid Command! - " + str(cmd) + "}}xx", log_level=1) # Only show commands if UAC active if is_admin[1]: commands = sorted(valid_commands.keys()) p("}}yn Valid Commands: " + str(commands) + "}}xx") p("}}ybFor help - type mgmt.exe help (command)}}xx") sys.exit(1) # Run the function associated w the command cmd_parts = valid_commands[cmd] if cmd_parts is None:
img /= 1160 img *= 255 points = centers[idx] img = util.draw_pose(dataset, img, lbl, 3, (255, 0, 0), points) cv2.imwrite( root_dir + 'samples/depth/' + phase + '_' + str(idx) + '.png', img) ############################################################################################## ############################# Normalize Labels ################################# ### Make sure centers are calculated using get_centers.py before running this ### Also make sure you are working with original non normalized 3D joint coordinates not the normalized ones. ### After running this segment you should rename the original and give this new file the original's name so that it becomes the labels file used moving forward ################################################################################ lbls = util.load_labels(dataset, phase) # load original test/train data centers = util.load_centers(dataset, phase).astype(float) fx, fy, ux, uy = util.get_param(dataset) out_file = root_dir + 'labels/fpad_' + phase + '_label_nm.txt' lbls = np.asarray([s.split() for s in lbls], dtype=np.float32) for lid, lbl in enumerate(lbls): joints = np.asarray(np.reshape(lbl, (21, 3)), dtype=np.float32) joints, skel_camcoords = util.world2pixel(joints, dataset) lbls[lid] = np.reshape(joints, (63)) lbls = np.reshape(lbls, (-1, 63)) x = util.normalize_pose(dataset, lbls, centers, 150, fx, fy) util.save_results(x, out_file) ################################################################################
def test(): global model count_test_samples = 0 correct = 0 falsePositives = 0 falseNegatives = 0 count_real_word = 0 count_predict_word = 0 word_label = [] word_pred = [] word_correct = 0 precision = 0 model.rnn.train(False) print("testing......") test_data_CL = load_data_tokenizer( test_path, len_chunk=args.len_lines_per_chunk, doShuffling=False, ) test_chars = create_tensor_classifier(test_data_CL) while True: try: numeric = next(test_chars) except StopIteration: break loss, numberOfCharacters, log_probs_cal, input_tensor_cal, target_tensor_cal = model.forward_cl( numeric, train=False) count_test_samples += args.batchSize tag_score = log_probs_cal flat_input = input_tensor_cal.transpose(1, 0) flat_label = target_tensor_cal.transpose(1, 0) tag_score = tag_score.transpose(0, 1) val, argmax = tag_score.max(dim=2) for b in range(args.batchSize): chars = [ itos[element.item()] if element.item() != 0 else "-" for element in flat_input[b] ] pred_seq = chars.copy() label = flat_label[b] pred = argmax[b] for i in range(args.sequence_length): if label[i] == 1: chars[i] = " " + chars[i] if pred[i] == 1: pred_seq[i] = " " + pred_seq[i] if pred[i] == label[i]: correct += 1 if pred[i] != label[i] and pred[i] == 0: falseNegatives += 1 if pred[i] != label[i] and pred[i] == 1: falsePositives += 1 # word level evaluation word_label.append(label[i]) word_pred.append(pred[i]) if pred[i] == 1: count_predict_word += 1 if label[i] == 1: if word_label == word_pred: word_correct += 1 word_pred = [] word_label = [] count_real_word += 1 if printAllPrediction and "".join(chars) != "".join(pred_seq): print("".join(chars)) print("".join(pred_seq)) if printPrediction and "".join(chars) != "".join(pred_seq): print("".join(chars)) print("".join(pred_seq)) print("train losses ", trainLosses) print("dev losses ", devLosses) print() print("============Evaluation===========") print() if correct == 0: print("correct predicted boundaries ", correct) print("correct predicted words ", word_correct) else: precision = correct / (correct + falsePositives) recall = correct / (correct + falseNegatives) f1 = 2 * (precision * recall) / (precision + recall) print("Boundary Measure") print("False Negatives ", falseNegatives) print("False Positives ", falsePositives) print("Correctly predicted boundaries", correct) print("Precision", round(precision * 100, 2), "Recall", round(recall * 100, 2), "F1", round(f1 * 100, 2)) print() print("Word Measure") word_precision = word_correct / (count_predict_word) word_recall = word_correct / (count_real_word) word_f1 = 2 * (word_precision * word_recall) / (word_precision + word_recall) print("Correctly predicted words", word_correct) print("Number of predicted words", count_predict_word) print("Number of real words", count_real_word) print("Precision", round(word_precision * 100, 2), "Recall", round(word_recall * 100, 2), "F1", round(word_f1 * 100, 2)) print() print("sample result") print("label : " + "".join(chars)) print("predict: " + "".join(pred_seq)) with open(log_path, "a+") as outFile: print("falseNegatives ", falseNegatives, file=outFile) print("falsePositives ", falsePositives, file=outFile) print("correct ", correct, file=outFile) print("Boundary measures: ", file=outFile) print("Precision", precision, "Recall", recall, "F1", 2 * (precision * recall) / (precision + recall), file=outFile) print("\nWord measures", "Precision", word_precision, "Recall", word_recall, "F1", 2 * (word_precision * word_recall) / (word_precision + word_recall), file=outFile) print("word_correct :", word_correct, file=outFile) print("count_predict_word: ", count_predict_word, file=outFile) print("count_real_word: ", count_real_word, file=outFile) print("", file=outFile) print("".join(chars), file=outFile) print("".join(pred_seq), file=outFile) print(total_time, file=outFile) csv_path = os.path.join(CHECKPOINTS_TOKENIZER, "tokenizer_result.csv") with open(csv_path, "a+") as table: print() print("..........................") print("-", file=table, end=';') if str(args.load_from)[:2] == "LM": print("LM", file=table, end=';') else: print("-", file=table, end=';') print(args.save_to, file=table, end=';') print(args.dataset, file=table, end=';') print("boundary", file=table, end=';') precision = round(precision * 100, 2) recall = round(recall * 100, 2) f1 = round(f1 * 100, 2) print(precision, file=table, end=';') print(recall, file=table, end=';') print(f1, file=table, end=';') print("word", file=table, end=';') word_precision = round(word_precision * 100, 2) word_recall = round(word_recall * 100, 2) word_f1 = round(word_f1 * 100, 2) print(word_precision, file=table, end=';') print(word_recall, file=table, end=';') print(word_f1, file=table, end=';') print(num_epoch + 1, file=table, end=';') print(total_time, file=table, end=';') print("trainLosses ", trainLosses, file=table, end=';') print("devLosses ", devLosses, file=table, end=';') print(args.add_note, file=table, end=';') p = util.get_param(str(args)) print(p, file=table, end=';') long, short = util.get_command(str(args)) p = "python Tokenizer.py " long = p + long print(long, file=table, end='\n')
def compute_layer_stats(layer): refreeze = False if hasattr(layer, 'frozen') and layer.frozen: u.unfreeze(layer) refreeze = True s = AttrDefault(str, {}) n = args.stats_batch_size param = u.get_param(layer) _d = len(param.flatten()) # dimensionality of parameters layer_idx = model.layers.index(layer) # TODO: get layer type, include it in name assert layer_idx >= 0 assert stats_data.shape[0] == n def backprop_loss(): model.zero_grad() output = model( stats_data) # use last saved data batch for backprop loss = compute_loss(output, stats_targets) loss.backward() return loss, output def backprop_output(): model.zero_grad() output = model(stats_data) output.backward(gradient=torch.ones_like(output)) return output # per-example gradients, n, d _loss, _output = backprop_loss() At = layer.data_input Bt = layer.grad_output * n G = u.khatri_rao_t(At, Bt) g = G.sum(dim=0, keepdim=True) / n u.check_close(g, u.vec(param.grad).t()) s.diversity = torch.norm(G, "fro")**2 / g.flatten().norm()**2 s.grad_fro = g.flatten().norm() s.param_fro = param.data.flatten().norm() pos_activations = torch.sum(layer.data_output > 0) neg_activations = torch.sum(layer.data_output <= 0) s.a_sparsity = neg_activations.float() / ( pos_activations + neg_activations) # 1 sparsity means all 0's activation_size = len(layer.data_output.flatten()) s.a_magnitude = torch.sum(layer.data_output) / activation_size _output = backprop_output() B2t = layer.grad_output J = u.khatri_rao_t(At, B2t) # batch output Jacobian H = J.t() @ J / n s.hessian_l2 = u.l2_norm(H) s.jacobian_l2 = u.l2_norm(J) J1 = J.sum(dim=0) / n # single output Jacobian s.J1_l2 = J1.norm() # newton decrement def loss_direction(direction, eps): """loss improvement if we take step eps in direction dir""" return u.to_python_scalar(eps * (direction @ g.t()) - 0.5 * eps**2 * direction @ H @ direction.t()) s.regret_newton = u.to_python_scalar(g @ u.pinv(H) @ g.t() / 2) # TODO: gradient diversity is stuck at 1 # TODO: newton/gradient angle # TODO: newton step magnitude s.grad_curvature = u.to_python_scalar( g @ H @ g.t()) # curvature in direction of g s.step_openai = u.to_python_scalar( s.grad_fro**2 / s.grad_curvature) if s.grad_curvature else 999 s.regret_gradient = loss_direction(g, s.step_openai) if refreeze: u.freeze(layer) return s
def approve_nic(): # Get the params for the nic nic_name = util.get_param(2) nic_network = util.get_param(3) if nic_name == "" or nic_network == "": p("}}rbError - Invalid paramaters! try mgmt.exe help approve_nic for more information}}xx" ) return False nic_id = None try: nic_id = int(nic_name) except: # Ok if this fails - trying to see if an ID was passed instead of a name pass if not nic_id is None: # Lookup the nic by ID iface = NetworkDevices.get_nic_by_interface_index(nic_id) if iface is None: # Unable to find an interface by this ID p("}}rbInvalid Interface ID! " + str(nic_id) + "}}xx") return False else: #nic_name = iface.Name # NOTE - Description will give us the driver name w/out the #2 after it nic_name = iface.Description # Strip off #? at the end of the name t_nic = nic_name removed_suffix = "" for i in range(1, 30): suffix = " #" + str(i) if suffix in t_nic: t_nic = t_nic.replace(suffix, "") removed_suffix = suffix if removed_suffix != "": p("}}ybNOTE - Stripped off " + removed_suffix + " from nic name\n - All instances of this nic approved with this network}}xx" ) nic_name = t_nic p("}}gnApproving " + nic_name + " on netowrk " + nic_network, log_level=1) # Get the list of approved nics # Load the value from the registry try: approved_nics_json = RegistrySettings.get_reg_value( app="OPEService", value_name="approved_nics", default="[]") nic_list = json.loads(approved_nics_json) # Add this nic to the list nic_list.append((nic_name, nic_network)) # Write this back to the registry approved_nics_json = json.dumps(nic_list) RegistrySettings.set_reg_value(app="OPEService", value_name="approved_nics", value=approved_nics_json) except Exception as ex: p("}}rbUnable to write approved nics to the registry!}}xx", log_level=1) return False # Force a reload of the device list NetworkDevices.init_device_list(refresh=True) return True
Author: Hengkai Guo, Xinghao Chen """ import numpy as np import matplotlib.pyplot as plt import util show_validate_figure = 0 # load hand poses labels = np.loadtxt('../labels/msra_test_label.txt') labels = np.reshape(labels, (-1, 21, 3)) # first convert uvd to xyz params = util.get_param('msra') labels = util.pixel2world(labels, *params) # wrist, mcp for index, middle, ring, little, thumb labels = labels[:, [0, 1, 5, 9, 13, 17], :] ''' The palm coordinate frame has its origin at wrist, positive Y axis pointing to the middle finger root and positive Z axis pointing outwards of the palm plane. Sun et al. "Cascaded Hand Pose Regression", CVPR 2015 ''' # y axis y = labels[:, 2, :] - labels[:, 0, :] # z axis
def start_upgrade_process(branch=None, force_upgrade=None): ret = True if not CredentialProcess.is_time_to_upgrade(): p("}}gnNot time to check for upgrades yet, skipping...}}xx", log_level=3) return True RegistrySettings.set_reg_value(value_name="last_upgrade_time", value=time.time()) curr_branch = branch if curr_branch is None: # See if a parameter was provided curr_branch = util.get_param(2, None) # Force upgrade - even if versions match if force_upgrade is None: force_upgrade = util.get_param(3, "") if force_upgrade.lower() != "-f": force_upgrade = False else: force_upgrade = True # If branch is still empty, get it from the registry if curr_branch is None: curr_branch = RegistrySettings.get_reg_value(value_name="install_branch", default="master") # Start by grabbing any new stuff from the git server ret = ProcessManagement.git_pull_branch(curr_branch) if ret is False: # Not critical if this fails - apply whatever is present if it is # a different version number # return False p("}}ybWARNING - Unable to pull updates for git server!}}xx") pass # Save the current branch for next time RegistrySettings.set_reg_value(value_name="install_branch", value=curr_branch) # Check the mgmt.version files to see if we have a new version ope_laptop_binaries_path = os.path.expandvars("%programdata%\\ope\\tmp\\ope_laptop_binaries") # Get the path to the mgmt.version file git_version_path = os.path.join(ope_laptop_binaries_path, "Services", "mgmt", "mgmt.version") # Do we have a new version? curr_version = CredentialProcess.get_mgmt_version() git_version = CredentialProcess.get_mgmt_version(git_version_path) if git_version == "NO VERSION": # No version file found p("}}ynNo version file found in git repo, skipping upgrade!}}xx") return False if not CredentialProcess.is_version_newer(curr_version, git_version) and force_upgrade is not True: # Same version - no upgrade needed p("}}gnOPE Software up to date: " + str(git_version) + " not newer than " + str(curr_version) + " - (not upgrading)}}xx") return True # Version is different, prep for update forced = "" if force_upgrade: forced = "}}yb(upgrade forced)}}gn" p("}}gnFound new version " + forced + " - starting upgrade process: " + \ curr_version + " --> " + git_version + "}}xx") # Lock user accounts if not UserAccounts.disable_student_accounts(): p("}}rbERROR - Unable to disable student accounts prior to upgrade!}}xx") return False # Make sure students are logged out if not UserAccounts.log_out_all_students(): p("}}rbERROR - Unable to log out student accounts prior to upgrade!}}xx") return False p("}}ynLaunching OPE Software Update process...}}xx") # run the upgrade_ope.cmd from the TMP rc folder!!! bat_path = os.path.join(ope_laptop_binaries_path, "Services\\mgmt\\rc\\upgrade_ope.cmd") # Add the redirect so we end up with a log file if not ProcessManagement.run_detatched_cmd(bat_path + " >> %programdata%\\ope\\tmp\\log\\upgrade.log 2>&1"): p("}}rbERROR - Unable to start upgrade process!}}xx") return False # Make sure to exit this app?? #sys.exit(0) return ret