def main(): dir_path = "./" log_path = dir_path + 'logs/m2/log.pkl' graph_path = dir_path + 'graphs/' # labels = ['w1', 'wd1', 'w2', 'wd2'] # train_losses = [None, None, None, None] # test_losses = [None, None, None, None] # test_accs = [None, None, None, None] # train_losses[0], test_losses[0], test_accs[0] = utils.read_log('logs/m1_ndrop/log.pkl', ([], [], [])) # train_losses[1], test_losses[1], test_accs[1] = utils.read_log('logs/m1/log.pkl', ([], [], [])) # train_losses[2], test_losses[2], test_accs[2] = utils.read_log('logs/m2_ndrop/log.pkl', ([], [], [])) # train_losses[3], test_losses[3], test_accs[3] = utils.read_log('logs/m2/log.pkl', ([], [], [])) labels = ['w1', 'w2', 'w3'] train_losses = [None, None, None] test_losses = [None, None, None] test_accs = [None, None, None] train_losses[0], test_losses[0], test_accs[0] = utils.read_log( 'logs/m1_ndrop/log.pkl', ([], [], [])) train_losses[1], test_losses[1], test_accs[1] = utils.read_log( 'logs/m2_ndrop/log.pkl', ([], [], [])) train_losses[2], test_losses[2], test_accs[2] = utils.read_log( 'logs/m3/log.pkl', ([], [], [])) train_losses[3], test_losses[3], test_accs[3] = utils.read_log( 'logs/m2/log.pkl', ([], [], [])) plot_multi_losses(train_losses, test_losses, test_accs, labels, graph_path)
def log_active_app_per_second(): now = datetime.now() log = read_log(year=now.year) seconds_in_memory_count = 0 while True: # Format: Day-Month-Year date_str = "{}-{}-{}".format(now.day, now.month, now.year) app_name = active_app_name() if date_str in log: if app_name in log[date_str]: log[date_str][app_name] += LOG_INTERVAL else: log[date_str][app_name] = LOG_INTERVAL else: log[date_str] = {} log[date_str][app_name] = LOG_INTERVAL # after LOG_INTERVAL seconds overwrite log file if seconds_in_memory_count >= MEMORY_SECONDS: update_log(log, year=now.year) now = datetime.now() log = read_log(year=now.year) seconds_in_memory_count = 0 else: seconds_in_memory_count += LOG_INTERVAL time.sleep(LOG_INTERVAL)
def anime_log(args): log_args = dict() if len(args) == 0: pass elif args[0].isnumeric(): log_args["number"] = int(args[0]) else: log_args["pattern"] = re.compile(args[0]) if len(args) == 2: log_args["number"] = int(args[1]) logs = utils.read_log(**log_args) ongoing = utils.read_log(logfile=config.ongoingfile) if len(logs) == 0: if len(args) == 0: outputs.warning_info("No log entries found.") else: outputs.prompt_val("Log entries not found for arguments", args[0], "error") return outputs.bold_info("Watched\t\tAnime Name") for k, log in logs.items(): outputs.normal_info(utils.Log(log).show(), end=" ") if k in ongoing: outputs.warning_tag("TRACKED", end="") outputs.normal_info()
def render_log(filename): sizes = [] computes = [] comms = [] sizes, comms, computes, merged_comms = read_log(filename) #sizes = sizes[::-1] #computes = computes[::-1] #comms = comms[::-1] start_time = 0.0 comm_start_time = 0.0 comm = 0.0 max_time = max(np.sum(computes), np.sum(comms)+computes[0]) fig, ax = plt.subplots(1) print('sizes: ', sizes) print('computes: ', computes) print('communications: ', comms) for i in range(len(computes)): comp = computes[i] bar = Bar(start_time, comp, max_time, ax, type='p') bar.render() if comm_start_time + comm > start_time + comp: comm_start_time = comm_start_time + comm else: comm_start_time = start_time + comp comm = comms[i] bar_m = Bar(comm_start_time, comm, max_time, ax, type='m') bar_m.render() start_time += comp plt.show() plt.clf() plt.scatter(sizes, comms, c='blue') plt.scatter(sizes, computes, c='red') plt.show()
def select_feature(log_name, n): """ 从日志中挑选前n个特征 :param log_name: :param n: :return: """ feature_names = [] log = read_log(log_name) fold_results = log['result']['fold_results'] zero_feature_names = [] for fold_result in fold_results: feats = [ feat_tuple[0] for feat_tuple in sorted( fold_result['feature_importance_dict'].items(), key=lambda item: item[1]) if feat_tuple[1] != 0 ] feature_names += feats[-n:] zero_feature_names += [ feat_tuple[0] for feat_tuple in sorted( fold_result['feature_importance_dict'].items(), key=lambda item: item[1]) if feat_tuple[1] == 0 ] feature_names = list(set(feature_names)) non_zero_feature_names = [] for feat in feature_names: if feat not in zero_feature_names: non_zero_feature_names.append(feat) print('The number of feature_names is ', len(feature_names)) return non_zero_feature_names
def main(file_path): if os.path.isdir(file_path): files = sorted(glob.glob(os.path.join(file_path, '*bo.log'))) log_dir = file_path else: files = [file_path] log_dir = os.path.dirname(file_path) fig_dir = os.path.join(log_dir, 'plots') if not os.path.exists(fig_dir): os.mkdir(fig_dir) layerwise_pruning_layers = raw_input( 'Plot layerwise pruning parameter layers: ') for i, log_file in enumerate(files): print('\nPlot', log_file) logs, constraint = read_log(log_file) prefix = os.path.join(fig_dir, os.path.basename(log_file)[:-4]) if constraint is None: constrained_bo = False else: constrained_bo = True print('Number of iterations:', len(logs)) if constraint is not None: print('Current constraint:', constraint) find_min_objective(logs, constraint, constrained_bo) plot_accuracy_latency(logs, constraint, constrained_bo, saturation=True, prefix=prefix) # plot_objective_time(logs, constraint, constrained_bo, prefix=prefix) if layerwise_pruning_layers == 'all' or str( i) in layerwise_pruning_layers: plot_layerwise_pruning_param(logs, prefix=prefix)
def completedefault(self, text, line, start, end): if re.match(r'([a-z-]+ +){2,}', line): return [] lists = set(utils.read_log().keys()).union( set(utils.read_cache(complete=True))) match = filter(lambda t: t.startswith(text), lists) return utils.completion_list(match)
def evaluation(): args = parse() class_label_dict, label_class_dict = _read_label_file(args.label) classes = list() num_classes = len(class_label_dict) for i in range(num_classes): classes.append(label_class_dict[i]) gt_dict = utils.read_json(args.gt, class_label_dict) pred_dict = utils.read_log(args.log, class_label_dict) y_true, y_pred = utils.gen_yture_ypred(gt_dict, pred_dict) metrics = utils.Metrics(y_true, y_pred, classes) conf_matrix = metrics.confusion_matrix metrics_list = metrics.metrics_list() # print result print("class evaluation") print("~" * 50) print("accuracy: %.6f" % (metrics_list[-1])) print("~" * 50) for i in range(num_classes): print("%s_recall: %.6f" % (label_class_dict[i], metrics_list[i * 2])) print("%s_precision: %.6f" % (label_class_dict[i], metrics_list[i * 2 + 1])) print("~" * 50) print("Confusion Matrix") print(conf_matrix) if args.verbose: metrics.plot_confusion_matrix() print("Done.")
def complete_url(self, text, line, *ignored): lists = set(utils.read_log().keys()).union( set(utils.read_cache(complete=True))) urls = map( lambda name: commands.anime_source_module.get_episode_url( name, ''), lists) match = filter(lambda t: t.startswith(text), urls) return utils.completion_list(match)
def anime_updates(anime_name=''): """Check and display the updates on the tracked anime list.""" if anime_name == '': anime_list = utils.read_log(logfile=config.ongoingfile) else: anime_list = { anime_name: utils.read_log(anime_name=anime_name, logfile=config.ongoingfile) } updates = {} for anime, episodes in anime_list.items(): new_episodes = gogoanime.get_episodes_range( gogoanime.get_anime_url(anime)) new = set(utils.extract_range(new_episodes)).difference( set(utils.extract_range(episodes))) if len(new) > 0: updates[anime] = new return updates
def complete_play(self, text, line, *ignored): m = re.match(r'[a-z-]+ +([0-9a-z-]+) +', line) if m: name = m.group(1) log = utils.read_log(name) if not log: return ["1"] ep = utils.Log(log).eps.split('-')[-1] return [str(int(ep) + 1)] return self.completedefault(text, line, *ignored)
def anime_updates(anime_name=""): """Check and display the updates on the tracked anime list.""" if anime_name == "": anime_list = utils.read_log(logfile=config.ongoingfile) else: anime_list = { anime_name: utils.read_log(anime_name=anime_name, logfile=config.ongoingfile) } updates = {} for anime, log in anime_list.items(): episodes = log.split()[1] new_episodes = anime_source_module.get_episodes_range( anime_source_module.get_anime_url(anime)) new = set(utils.extract_range(new_episodes)).difference( set(utils.extract_range(episodes))) if len(new) > 0: updates[anime] = new return updates
def untrack_anime(anime_name): """Remove an anime from the track list""" anime_list = utils.read_log(logfile=config.ongoingfile) if anime_name in anime_list: anime_list.pop(anime_name) else: return with open(config.ongoingfile, 'w') as w: for k, v in anime_list.items(): w.write(f'{k} {v}\n')
def main(args): df = utils.read_log(args.input) # df = df.iloc[:1100] # df['heat_on'] = (df['time'] < 700.) * df['heat_on'] # df['temperature'] += 273. x0 = {'k': 0.17, 'c_v': 3000., 'volume': 1., 'u_0': 300., 'h': 2.2, 'wattage': 800, 'u_env': 298., 'init_time_off': 80.} fixed = {} output_path = '%s.params' % args.input if os.path.exists(output_path): print "reading params from: %s" % output_path with open(output_path, 'r') as f: optimal = simplejson.load(f) else: optimal = utils.optimal_parameters(df['time'].values, df['heat_on'].values, df['temperature'].values, params=x0, sig=0.1, **fixed) optimal.update(fixed) with open(output_path, 'w') as f: simplejson.dump(optimal, f, sort_keys=True, indent=2) print utils.pretty_params(optimal) short = df.iloc[:250] heat = control.control_params(320, short['time'].values, short['heat_on'].values, short['temperature'].values, optimal) u_samp = np.array(list(utils.compute_temperature(df['time'].values, df['heat_on'].values, **optimal))) plt.plot(df['time'], u_samp) plt.plot(df['time'], df['temperature']) ax = plt.twinx() ax.fill_between(df['time'], 0., df['heat_on'], color='r', alpha=0.5) ax.set_ylim([-0.1, 1.1]) plt.show() import ipdb; ipdb.set_trace()
def continue_play(args, play_func=play_anime): name, _ = read_args(args, episodes=False) log = utils.Log(utils.read_log().get(name)) watch_later = utils.read_log(name, logfile=config.watchlaterfile) if watch_later: episodes = utils.extract_range(utils.Log(watch_later).eps) else: _, episodes = read_args(args) outputs.prompt_val("Watched", log._eps, "success", end='\t') outputs.normal_info(log.last_updated_fmt) if not log.eps: last = 0 else: last = int(re.split('-|,', log.eps)[-1]) to_play = utils.compress_range(filter(lambda e: e > last, episodes)) if to_play.strip(): play_func([name, to_play]) else: unsave_anime(name)
def continue_play(args): name, episodes = read_args(args) watched = utils.read_log().get(name) outputs.prompt_val('Watched', watched, 'success') if not watched: last = 0 else: last = int(watched.split('-')[-1]) play_anime( [name, utils.compress_range(filter(lambda e: e > last, episodes))])
def list_episodes(args): name = read_args(args, episodes=False) available_rng = gogoanime.get_episodes_range(gogoanime.get_anime_url(name)) if len(args) == 2: _, episodes = read_args(args) eps = set(episodes) avl_eps = set(utils.extract_range(available_rng)) res = eps.intersection(avl_eps) available_rng = utils.compress_range(res) outputs.prompt_val(f'Available episodes', available_rng) outputs.prompt_val(f'Watched episodes', utils.read_log(name), 'success') utils.write_cache(name)
def parse_real_comm_cost(): configs = ['GoogleNet', 'gm'] #SyncEASGD name = configs[0] t = configs[1] nnodes = [2, 4, 8] ncomms = [] for n in nnodes: test_file = '/home/shshi/gpuhome/repositories/dpBenchmark/tools/caffe/cnn/%s/%s%dcomm.log' % ( name.lower(), t, n) sizes, comms, computes, merged_comms = read_log(test_file) ncomms.append(np.sum(merged_comms)) print('network: ', name, ', type: ', t) print('ncomms: ', ncomms)
def list_episodes(args): name, _ = read_args(args, episodes=False) available_rng = anime_source_module.get_episodes_range(anime_source_module.get_anime_url(name)) if len(args) == 2: _, episodes = read_args(args) eps = set(episodes) avl_eps = set(utils.extract_range(available_rng)) res = eps.intersection(avl_eps) available_rng = utils.compress_range(res) outputs.prompt_val("Available episodes", available_rng) log = utils.Log(utils.read_log(name)) outputs.prompt_val("Watched episodes", log.eps, "success", end=' ') outputs.normal_info(log.last_updated_fmt) utils.write_cache(name)
def anime_log(args): outputs.bold_info('Watched\t\tAnime Name') log_args = dict() if len(args) == 0: pass elif args[0].isnumeric(): log_args['number'] = int(args[0]) else: log_args['pattern'] = re.compile(args[0]) if len(args) == 2: log_args['number'] = int(args[1]) logs = utils.read_log(**log_args).items() for k, v in logs: outputs.normal_info(f'{v}\t\t{k}')
def verify_anime_exists(anime_name, verbose=False): if utils.read_log(anime_name) is not None: if verbose: outputs.normal_info(anime_name, 'LOG', reverse=True) return True elif anime_name in utils.read_cache(complete=True): if verbose: outputs.normal_info(anime_name, 'CACHE', reverse=True) return True elif utils.get_soup(get_anime_url(anime_name)) is not None: if verbose: outputs.normal_info(anime_name, 'SITE', reverse=True) return True else: return False
def track_anime(args): """Put an anime into the track list""" anime_name = read_args(args, episodes=False) watched_episodes = utils.read_log(anime_name) if watched_episodes is None: outputs.warning_info( 'Log entry not found. Setting only new episodes for tracking.') _, episodes = read_args(args) episodes = utils.compress_range(episodes) else: outputs.prompt_val(f'Watched', watched_episodes, 'success') episodes = watched_episodes utils.write_log(anime_name, episodes, append=False, logfile=config.ongoingfile)
def save_anime(args): """Put the anime into watch later list.""" anime_name, eps = read_args(args) watched = utils.read_log(anime_name) if watched: watched_eps = utils.extract_range(utils.Log(watched).eps) else: watched_eps = [] save_eps = set(eps).difference(set(watched_eps)) if not save_eps: outputs.warning_info('Already watched the provided episodes.') return utils.write_log(anime_name, utils.compress_range(save_eps), append=True, logfile=config.watchlaterfile)
def combine_feature_names(log_names, feat_hist_file_name, batch_names): """ :param log_names: list, 日志名列表 :param feat_hist_file_name: str, 特征生成历史文件名 :param batch_names: list, 批次名列表 :return: """ old_feature_names = [] # 基础特征 for log_name in log_names: log = read_log(log_name) feats = log['config']['feature_names'] old_feature_names += feats # 添加需要被测试的新特征 new_feature_names = read_feature_names_from_hist(feat_hist_file_name, batch_names) return old_feature_names, new_feature_names
def track_anime(args): """Put an anime into the track list""" anime_name, episodes = read_args(args, episodes=False) log = utils.read_log(anime_name) if log is None: outputs.warning_info( "Log entry not found.") if not episodes: _, episodes = read_args(args) episodes = utils.compress_range(episodes) else: episodes = utils.Log(log).eps outputs.prompt_val("Watched", episodes, "success") utils.write_log(anime_name, episodes, append=False, logfile=config.ongoingfile)
def gmwfbp_simulate(): name = 'GoogleNet' #name = 'ResNet' #name = 'VGG' #name = 'DenseNet' num_of_nodes = 32 test_file = '/media/sf_Shared_Data/gpuhome/repositories/dpBenchmark/tools/caffe/cnn/%s/tmp8comm.log' % name.lower( ) sizes, comms, computes, merged_comms = read_log(test_file) #computes = [c/4 for c in computes] #sizes = [1., 1., 1., 1.] #computes = [3., 3.5, 5., 6.] #sim = Simulator(name, computes[0:4], sizes[0:4], num_of_nodes) sim = Simulator(name, computes, sizes, num_of_nodes) #sim.wfbp() sim.gmwfbp2() plt.savefig('%s/breakdown%s.pdf' % (OUTPUT_PATH, name.lower()))
def learn_daemon(log_path, timeout): param_path = '%s.params' % log_path params = parameters.pot_of_water(0.1, 0.05) while True: df = utils.read_log(log_path) params = utils.optimal_parameters(df['time'].values, df['heat_on'].values, df['temperature'].values, params=params, sig=0.1) lock_path = '%s.lock' % param_path lock = filelock.FileLock(lock_path, timeout=timeout) with lock.acquire(): with open(param_path, 'w') as f: simplejson.dump(params, f, sort_keys=True, indent=2)
def statastic_gradient_size(filename, label, color, marker): global ax sizes, comms, computes, merged_comms = read_log(filename) if ax is None: fig, ax = plt.subplots(figsize=(5,4.5)) fontsize = 14 ax.scatter(range(1, len(sizes)+1), sizes, c=color, label=label, marker=marker, s=40, facecolors='none', edgecolors=color) #plot_hist(sizes) ax.set_xlim(left=0) ax.set_xlabel('Learnable layer ID') #plt.ylim(bottom=1e3, top=1e7) #plt.ylabel('Message size (bytes)') ax.set_ylabel('# of parameters') ax.set_yscale("log", nonposy='clip') ax.legend() update_fontsize(ax, fontsize) print('total size: ', np.sum(sizes)) return sizes
def index(self): balance, position = self.p.portfolio_info() text = f'Daxiang Trading Robot - Uptime {t.now() - self.start_time} <br><hr>' text += f'Current Position: {position[0]}, Average Price: {position[1]} <br>' text += 'Profit History: <br>' text += '---------------------- Balance(XBT) -- Change_Rate -- Total_Change_Rate<br>' for b in balance: text += f'{b[0]}: {b[1]/100000000}, {b[2]}%, {b[3]}% <br>' text += '<br><hr>' text += 'Recent System Log: <br>' text += u.read_log('log/daxiang_robot.log') text += '<hr>' text += 'History System Log: <br>' for file in os.listdir('log'): if file.startswith("daxiang_robot"): text += u.href_wrapper(file) text += '<br>' return text
def control_daemon(log_path, heat_pin, timeout): param_path = '%s.params' % log_path while True: df = utils.read_log(log_path, timeout=timeout) params = utils.read_params(param_path, timeout=timeout) heat_on = control_params(320, df['time'].values, df['heat_on'].values, df['temperature'].values, params) import ipdb; ipdb.set_trace() if heat_on: GPIO.output(heat_pin, GPIO.HIGH) else: GPIO.output(heat_pin, GPIO.LOW)
def main(args): x0 = {'k': 0.17, 'c_v': 3000., 'volume': 1., 'u_0': 300., 'h': 2.2, 'wattage': 800, 'u_env': 298., 'init_time_off': 80.} fixed = {} output_path = args.output or '%s.params' % args.input while True: df = utils.read_log() df = df.iloc[:1100] df['heat_on'] = (df['time'] < 700.) * df['heat_on'] df['temperature'] += 273. if os.path.exists(output_path): logger.info("Reading params from: %s" % output_path) with open(output_path, 'r') as f: prev_optimal = simplejson.load(f) initial = {k: prev_optimal[k] for k in x0.keys()} else: initial = x0 optimal = utils.optimal_parameters(df['time'].values, df['heat_on'].values, df['temperature'].values, params=initial, sig=0.1, **fixed) optimal.update(fixed) with open(output_path, 'w') as f: simplejson.dump(optimal, f, sort_keys=True, indent=2) logging.info(utils.pretty_params(optimal))
def validate_between_versions(log1, v2_name): """ 假设使用版本 v1_name 本地交叉验证过一次,其log为log1, 利用这个Log读取模型,预测v2_name特征,并计算分数 :param log1: :param v2_name: :return: """ # 模型等相关数据准备 log = read_log(log1) feature_names = log['config']['feature_names'] config_name = log['config']['config_name'] model_name = log['config']['model']['model_name'] best_iteration = log['result']['best_iteration'] num_model = len(best_iteration) models = read_model_file(config_name, model_name, num_model) # 读取v2_name 版本的特征 Xtrain, Ytrain, Xtest = CombineFeature(feature_names=feature_names, versions=[v2_name], test=False) if model_name == 'xgboost': Xtrain = xgb.DMatrix(Xtrain[feature_names].values, feature_names=feature_names) # 预测 submission_list = [] for model, best_iter in zip(models, best_iteration): if model_name == 'xgboost': submission_list.append(model.predict(Xtrain, ntree_limit=best_iter)) elif model_name == 'lightgbm': submission_list.append( model.predict(Xtrain[feature_names].values, num_iteration=best_iter)) y_pred = np.mean(submission_list, axis=0) auc = roc_auc_score(Ytrain['label'], y_pred) print('auc is {}'.format(auc)) return auc
from models import DraftNet from utils import read_log import sys #get logfile name from command line log_file = sys.argv[1] #get model and card mapping model = torch.load('Saved_Models/draft_model.pkl') card_df = torch.load('Data/ft.pkl') card_map = card_df['name'].to_dict() #read logfile pools, picks, packs = read_log(log_file, card_df) #turn log to proper format create_data = [ torch.cat([torch.tensor(pools[i]), torch.tensor(packs[i])]) for i in range(len(packs)) ] data = torch.stack(create_data) #make prediction prediction = model(data.type(torch.float32)) #compare prediction with actual user pick for i, idx in enumerate(res.argsort(1, descending=False)): actual = picks[i].argmax() #only display top 3 if not at the end of the pack if 14 - ((i % 14) + 1) > 2: pred = idx[[-1, -2, -3]] else: pred = idx[[-1]] pstr = ", ".join([card_map[p.item()] for p in pred]) print("Pick ", i, ":")
if len(original_images) < 5: original_images.append(data[0].cpu()) rect_images.append(output[0].cpu()) save_image(original_images + rect_images, output_dir + '.png', padding=0, nrow=len(original_images)) average_loss = heldout_loss / len(heldout) print("Total heldout loss: " + heldout_loss) print("Average heldout loss: " + average_loss) # Generate images using the model and save them. samples = generate(model, 25, device) save_image(samples, output_dir + 'reconstructions.png', padding=0, nrow=5) train_losses, test_losses = utils.read_log(log_dir, ([], [])) #plot_loss(train_losses, test_losses, PLOT_PATH) # Get images with the desired properties. # Interpolate between images latent arithmetic. #im1_z = get_z(im1, model, device) #im2_z = get_z(im2, model, device) #sunglass_z = get_average_z(man_sunglasses, model, device) - get_average_z(man, model, device) #arith1 = latent_arithmetic(man_z, sunglass_z, model, device) #arith2 = latent_arithmetic(woman_z, sunglass_z, model, device) #save_image(arith1 + arith2, OUTPUT_PATH + 'arithmetic-dfc' + '.png', padding=0, nrow=10) # Linear interpolate #inter1 = linear_interpolate(man[0], man[1], model, device)
parameters = PARAMETERS[sensor] if verbose: print "found %d records" % len(measurements) flt_meas, flt_idx = utils.filter_meas(measurements, parameters.noise_window, parameters.noise_threshold) if verbose: print "remaining %d after low pass" % len(flt_meas) p0 = utils.get_min_max_guess(flt_meas, parameters.sensor_ref) cp0, np0 = utils.scale_measurements(flt_meas, p0) print "initial guess : avg %f std %f" % (np0.mean(), np0.std()) def err_func(p,meas,y): cp, np = utils.scale_measurements(meas, p) err = y*scipy.ones(len(meas)) - np return err p1, success = scipy.optimize.leastsq(err_func, p0[:], args=(flt_meas, parameters.sensor_ref)) cp1, np1 = utils.scale_measurements(flt_meas, p1) print "optimized guess : avg %f std %f" % (np1.mean(), np1.std()) utils.print_xml(p1, sensor, parameters.sensor_res) print "" utils.plot_results(measurements, flt_idx, flt_meas, cp0, np0, cp1, np1, parameters.sensor_ref) if __name__ == "__main__": sensor = "ACCEL" meas = utils.read_log("IMU_ACCEL_RAW.log", sensor) calibrate_sensor(sensor, meas, True)