def read_index(args): file_path = os.path.join(os.getcwd(), 'data') if (args.index == 1): file = os.path.join(file_path, '000001.csv') title = "SSE Composite Index" elif (args.index == 2): file = os.path.join(file_path, '399001.csv') title = "SZSE Component Index" elif (args.index == 3): file = os.path.join(file_path, '000016.csv') title = "SSE 50 Index" elif (args.index == 4): file = os.path.join(file_path, '399300.csv') title = "CSI 300 Index" else: file = os.path.join(file_path, '000905.csv') title = "CSI Smallcap 500 index" data = [] time = [] with open(file, 'r') as fp: reader = csv.reader(fp) next(reader) for line in reader: times = line[0] index = line[3] data.append(float(index)) time.append(times) # One year include about 244 trading days # I only choose the last 5 years data to make demo timeseries = 244 * 5 j = timeseries INDEX = [] Time_Label = [] while (j >= 0): INDEX.append(round(data[j], 2)) temp = time[j].split("/") newtime = '-'.join(temp) Time_Label.append(newtime) j -= 1 data = {"time": Time_Label, "index": INDEX} data = DataFrame(data) print(data.head()) # Draw the picture of INDEX name = title.replace(' ', '-') + '.png' path = os.path.join(os.getcwd(), 'pics') plot(data, path=os.path.join(path, name), title=title) print("Curve of {} has saved in {}\n".format(title, os.path.join(path, name))) return data, title
def main(): # read data lidar_file_name = "lidar/train_lidar2" joint_file_name = "joint/train_joint2" lidar_data = get_lidar(lidar_file_name) joint_data = get_joint(joint_file_name) robot_pose = get_odom(lidar_data) lidar_data = joint_align(lidar_data, joint_data) # particles init P = particleInit(num=128) # grid map init MAP = mapInit() # init parameters step_size = 20 trajectory = np.empty(shape=(1, 2)) var_scale = np.array([0.001, 0.001, 0.01 * np.pi / 180]) lidar_angles = np.arange(-135, 135.25, 0.25) * np.pi / 180 for i in range(0, len(lidar_data), step_size): if (i % 100 == 0): print(i) ''' Predict ''' delta_pose = get_motion(robot_pose, lidar_data[i], i, step_size) P['states'] = motion_model_predict(P['states'], delta_pose, var_scale) ''' Update ''' best_particle = measurement_model_update(MAP, P, lidar_data[i], lidar_angles) trajectory = np.vstack((trajectory, [ int(best_particle[0] / MAP['res']) + MAP['sizex'] // 2, int(best_particle[1] / MAP['res']) + MAP['sizey'] // 2 ])) ''' Mapping ''' MAP['map'] = mapping(MAP['map'], lidar_data[i], best_particle, MAP['res'], lidar_angles) ''' Resample ''' N_eff = 1 / np.sum(P['weight']**2) if N_eff < 0.3 * P['number']: print("Resampling...") P = resampling_wheel(P) # plot plot(MAP['map'], MAP['res'], robot_pose, trajectory)
def plotMergedTopPartons(era, result_file, verbose): if verbose: print "{0}: {1}".format(era, result_file) f = ROOT.TFile(result_file, "read") # FatJet_nGenPart histograms h_TTbar_MergedTop_nGenPart_nmteq1 = f.Get("FatJet_nGenPart/TTbar_MergedTop_nGenPart_recalc_nmteq1FatJet_nGenPartFatJet_nGenPart{FatJet_Stop0l=1}TTbar recalcsingle") h_ZNuNu_MergedTop_nGenPart_nmteq1 = f.Get("FatJet_nGenPart/ZNuNu_MergedTop_nGenPart_recalc_nmteq1FatJet_nGenPartFatJet_nGenPart{FatJet_Stop0l=1}ZJetsToNuNu recalcsingle") h_GJets_MergedTop_nGenPart_nmteq1 = f.Get("FatJet_nGenPart/GJets_MergedTop_nGenPart_recalc_nmteq1FatJet_nGenPartFatJet_nGenPart{FatJet_Stop0l=1}GJets recalcsingle") histograms = [h_TTbar_MergedTop_nGenPart_nmteq1, h_ZNuNu_MergedTop_nGenPart_nmteq1, h_GJets_MergedTop_nGenPart_nmteq1] labels = ["TTbar", "ZNuNu", "GJets"] #histograms = [h_ZNuNu_MergedTop_nGenPart_nmteq1, h_GJets_MergedTop_nGenPart_nmteq1] #labels = ["ZNuNu", "GJets"] name_1 = "MergedTop_nGenPart_nmteq1" name_2 = "MergedTop_nGenPart_nmteq1_norm" title_1 = "MergedTop_nGenPart, nmt=1, {0}".format(era) title_2 = "MergedTop_nGenPart, nmt=1, {0}, norm.".format(era) x_title = "MergedTop_nGenPart" y_title = "Events" x_min = 0 x_max = 11 # y limits for TTbar, ZNuNu, GJets y_min_1 = 10**-1 y_max_1 = 10**5 y_min_2 = 0 y_max_2 = 0.8 # y limits for ZNuNu, GJets #y_min_1 = 0 #y_max_1 = 300 #y_min_2 = 0 #y_max_2 = 0.3 # WARNING: if using setLog=True, do not use y_min = 0 # WARNING: currently stats do not show properly on log scale... this would need work to fix # plot(histograms, labels, name, title, x_title, y_title, x_min, x_max, y_min, y_max, era, plot_dir, showStats=False, normalize=False, setLog=False) plot(histograms, labels, name_1, title_1, x_title, y_title, x_min, x_max, y_min_1, y_max_1, era, "more_plots", showStats=False, normalize=False, setLog=True) plot(histograms, labels, name_2, title_2, x_title, y_title, x_min, x_max, y_min_2, y_max_2, era, "more_plots", showStats=True, normalize=True)
if __name__ == '__main__': filenameX = os.path.join(path, 'X.h5') filenameY = os.path.join(path, 'Y.h5') r = reader.H5Reader(data_shape={'Y': (None, 5)}) dataX = r.read(filenameX) dataY = r.read(filenameY) X = dataX['X'] Y = dataY['Y'] # keras model model = algo.lasso(alpha=.001) model.fit(X, Y) # bayesian regression # model = algo.create_model_linear( X.shape[1], Y.shape[1] ) # model.fit( X, Y, batch_size=100, epochs=1000 ) Yr = model.predict(X) print('MAE in-sample: ', tools.mae(Y, Yr)) print('MSE in-sample: ', tools.mse(Y, Yr)) print('MAPE in-sample: ', tools.mape(Y, Yr)) tools.plot(Yr) tools.QQplot(Yr, Y) tools.boxplot(Yr)
if argres[i] == arg1[i]: if arg1[i] == arg2[i]: scrres[i] += 1.6 else: scrres[i] += 0.8 elif arg1[i] == arg2[i]: print(scr1[i], scr2[i]) if (scr1[i] > 0.4 or scr2[i] > 0.4): if (scrres[i] < 0.7): argres[i] = arg1[i] if argt5[i][1] == arg1[i]: scrres[i] = 0.85 + 0.001 * (scr1[i] + scr2[i]) else: scrres[i] = 0.8 + 0.001 * (scr1[i] + scr2[i]) # elif scr1[i]>0.5: # argres[i] = arg1[i] # scrres[i] = 0.82+scr1[i] print(len(argres)) print(len(f2list)) #eval #get max score and label lbs = tools.getLabel(d, argres) truthlist = tools.getTruthlist(lbs, f2list) print('Plotting...') srt = tools.getSorted(truthlist, scrres) tools.plot(srt) tablelist = tools.sortTable(f2list, lbs, scrres, truthlist) table.getTable(tablelist)
abs(v - V[s])) # assign the change in value per iteration to delta if delta < theta: break # if change gets to negligible # --> converged to optimal value pi = np.zeros((env.env.nS, env.env.nA)) action = np.zeros((env.env.nS)) for s in range(env.env.nS): pi = argmax(env, V, pi, action, s, gamma) # extract optimal policy using action value return V, pi, action # optimal value funtion, optimal policy V, pi, action = value_iteration(env, gamma, theta) tools.plot(V, pi) a = np.reshape(action, (4, 4)) print(a) # discrete action to take in given state e = 0 for i_episode in range(100): c = env.reset() for t in range(10000): c, reward, done, info = env.step(action[c]) if done: if reward == 1: e += 1 break print( " agent succeeded to reach goal {} out of 100 Episodes using this policy ".
def run_covering(args, data, title): start = args.start_time col = data[data['time'].isin([start])] col = col["index"].tolist() if (len(col) == 0): print( "Your start time is not a trading day. So I postpone it to the next trading day.\n" ) while (len(col) == 0): start = next_day(start) col = data[data['time'].isin([start])] start_idd = col.index.tolist()[0] end_idd = start_idd + 244 * args.duration if (end_idd > 1220): end_idd = 1220 start_time = str(data["time"][data.index[start_idd]])[:-9] end_time = str(data["time"][data.index[end_idd]])[:-9] shares = [] values = [] pays = [] yield_time = [] i = 1 while (start_idd <= end_idd): if len(shares) == 0: share = args.money / data['index'][data.index[start_idd]] shares.append(share) yield_time.append(data["time"][data.index[start_idd]]) pays.append(args.money) values.append(args.money) else: value = sum(shares) * data['index'][data.index[start_idd]] pay = args.money * i - value share = pay / data['index'][data.index[start_idd]] yield_time.append(data["time"][data.index[start_idd]]) shares.append(share) pays.append(pay) values.append(args.money * i) i += 1 start_idd += args.interval return_value = values[-1] pay_value = sum(pays) yields = [] for i in range(len(pays)): yield_rate = (values[i] - sum(pays[:i + 1])) / sum( pays[:i + 1]) * 100.0 yields.append(yield_rate) max_yield = max(yields) data = {"time": yield_time, "index": yields} data = DataFrame(data) path = os.path.join(os.getcwd(), 'pics') name = "Yield-of-" + title.replace(' ', '-') + '.png' plot(data, path=os.path.join(path, name), ylabel="yield", title="Yield of " + title, freq="30D") print("Your strategy of investment is {}\n".format(args.strategy)) print(" Start investment at {}".format(start_time)) print(" End investment at {}".format(end_time)) print(" Investment duration is {} year(s)".format(args.duration)) print(" Investment interval is {} trading day(s)".format( args.interval)) print(" Captial investment is {:.2f} RMB\n".format(args.money)) print(" Your captial money is {:.2f} RMB".format(pay_value)) print( " Your ruturn of investment is {:.2f} RMB\n".format(return_value)) print(" Your final yield is {:.2f}%".format(yields[-1])) print(" Your annual yield is {:.2f}%\n".format( annual_rate((1 + yields[-1] / 100), args.duration) * 100)) print(" Your final max yield is {:.2f}%".format(max_yield)) print(" Your annual max yield is {:.2f}%".format( annual_rate((1 + max_yield / 100), args.duration) * 100)) print(" Your yield curve has saved in {}".format( os.path.join(path, name)))
def run(era): verbose = 2 print "---------- Running {0} ----------".format(era) # for datacard, the allowed Sgamma range is [0.01, 5] minSgamma = 0.01 maxSgamma = 5.0 # sufficient data to calclate shape factor minData = 5 fileSB = "results/SearchBinResults.json" fileYields = "datacard_inputs/zinv_yields_" + era + ".json" fileBinMap = "dc_BkgPred_BinMaps_master.json" with open(fileSB, "r") as f: sbResults = json.load(f) with open(fileYields, "r") as f: yieldResults = json.load(f) with open(fileBinMap, "r") as f: binMap = json.load(f) # histograms nbins_1 = 50 nbins_2 = 60 limits_1 = [-5, 30] limits_2 = [-1, 5] # multiple sets of histograms with different limits h_sgamma_searchbins_1 = ROOT.TH1F("h_sgamma_searchbins_1", "h_sgamma_searchbins_1", nbins_1, limits_1[0], limits_1[1]) h_sgamma_crunits_1 = ROOT.TH1F("h_sgamma_crunits_1", "h_sgamma_crunits_1", nbins_1, limits_1[0], limits_1[1]) h_sgamma_searchbins_2 = ROOT.TH1F("h_sgamma_searchbins_2", "h_sgamma_searchbins_2", nbins_2, limits_2[0], limits_2[1]) h_sgamma_crunits_2 = ROOT.TH1F("h_sgamma_crunits_2", "h_sgamma_crunits_2", nbins_2, limits_2[0], limits_2[1]) # --------------------------------------------- # # get bin and sgamma values for each search bin # # --------------------------------------------- # sgammaForSearchBins = [] searchBinAndSgamma = list((int(b), sbResults[era][b]["shape"]) for b in sbResults[era]) searchBinAndSgamma.sort(key = lambda x: x[0]) for x in searchBinAndSgamma: b = x[0] sgamma = x[1] sgammaForSearchBins.append(sgamma) h_sgamma_searchbins_1.Fill(sgamma) h_sgamma_searchbins_2.Fill(sgamma) if verbose > 1 and (sgamma < minSgamma or sgamma > maxSgamma): print "search bin {0}, sgamma = {1}".format(b, sgamma) # ----------------------------------------------------- # # get bin and sgamma values for each control region bin # # ----------------------------------------------------- # total_phocr_data = 0 total_phocr_gjets = 0 total_phocr_back = 0 sgammaForCRUnits = [] sgammaForCRUnitsGoodData = [] CRBinNames = {} for binName in binMap["unitCRNum"]["phocr"]: b = int(binMap["unitCRNum"]["phocr"][binName]) CRBinNames[b] = binName # keys for yieldResults["yieldsMap"]: # znunu # phocr_back # phocr_data # phocr_gjets for b in xrange(len(CRBinNames)): binName = CRBinNames[b] phocr_data = yieldResults["yieldsMap"]["phocr_data"][binName][0] phocr_gjets = yieldResults["yieldsMap"]["phocr_gjets"][binName][0] phocr_back = yieldResults["yieldsMap"]["phocr_back"][binName][0] # add to totals total_phocr_data += phocr_data total_phocr_gjets += phocr_gjets total_phocr_back += phocr_back sgamma = -999 den = phocr_gjets + phocr_back if den > 0.0: sgamma = phocr_data / den else: print "WARNING: CR bin {0}, denominator = {1}".format(b, den) sgammaForCRUnits.append(sgamma) h_sgamma_crunits_1.Fill(sgamma) h_sgamma_crunits_2.Fill(sgamma) # sgamma values for bins with sufficient number of data events if phocr_data >= minData: sgammaForCRUnitsGoodData.append(sgamma) if verbose > 1 and (sgamma < minSgamma or sgamma > maxSgamma): print "CR bin {0}, sgamma = {1}; phocr_data = {2}, phocr_gjets = {3}, phocr_back = {4}".format(b, sgamma, phocr_data, phocr_gjets, phocr_back) if verbose > 2 and phocr_data <= 2.0: print "CR bin {0}: phocr_data = {1}".format(b, phocr_data) if verbose > 2 and phocr_gjets <= 2.0: print "CR bin {0}: phocr_gjets = {1}".format(b, phocr_gjets) # total normalization total_norm = total_phocr_data / (total_phocr_gjets + total_phocr_back) histograms_1 = [h_sgamma_searchbins_1, h_sgamma_crunits_1] histograms_2 = [h_sgamma_searchbins_2, h_sgamma_crunits_2] labels = ["sgamma_searchbins", "sgamma_crunits"] # --- plot --- # # plot(histograms, labels, name, title, x_title, y_title, x_min, x_max, y_min, y_max, era, plot_dir, showStats=False, normalize=False, setLog=False) plot(histograms_1, labels, "sgamma_binning1", "Sgamma for " + era, "Sgamma", "Events", limits_1[0], limits_1[1], 0.0, 200.0, era, "more_plots", showStats=True) plot(histograms_2, labels, "sgamma_binning2", "Sgamma for " + era, "Sgamma", "Events", limits_2[0], limits_2[1], 0.0, 60.0, era, "more_plots", showStats=True) if verbose > 0: print "Total data = {0}".format(total_phocr_data) print "Total gjets = {0}".format(total_phocr_gjets) print "Total other background = {0}".format(total_phocr_back) print "Total normalization: data / (gjets + other back) = {0}".format(total_norm) print "Sgamma in search bins ({0}): mean = {1:.2f}, std_dev = {2:.2f}, min = {3:.2f}, max = {4:.2f}".format(len(sgammaForSearchBins), np.mean(sgammaForSearchBins), np.std(sgammaForSearchBins), np.amin(sgammaForSearchBins), np.amax(sgammaForSearchBins)) print "Sgamma in control bins ({0}): mean = {1:.2f}, std_dev = {2:.2f}, min = {3:.2f}, max = {4:.2f}".format(len(sgammaForCRUnits), np.mean(sgammaForCRUnits), np.std(sgammaForCRUnits), np.amin(sgammaForCRUnits), np.amax(sgammaForCRUnits)) print "Sgamma in control bins with at least {0} data events ({1}): mean = {2:.2f}, std_dev = {3:.2f}, min = {4:.2f}, max = {5:.2f}".format(minData, len(sgammaForCRUnitsGoodData), np.mean(sgammaForCRUnitsGoodData), np.std(sgammaForCRUnitsGoodData), np.amin(sgammaForCRUnitsGoodData), np.amax(sgammaForCRUnitsGoodData)) del h_sgamma_searchbins_1 del h_sgamma_searchbins_2 del h_sgamma_crunits_1 del h_sgamma_crunits_2
def train_model(model, criterion, optimizer, scheduler, dataset_sizes, num_epochs=25): since = time.time() best_model_wts = copy.deepcopy(model.state_dict()) best_acc = 0.9660 loss_p = {'train': [], 'val': []} acc_p = {'train': [], 'val': []} for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': model.train() # Set model to training mode else: model.eval() # Set model to evaluate mode running_loss = 0.0 running_corrects = 0 # Iterate over data. for inputs, labels in dataloaders[phase]: inputs = inputs.to(DEVICE) labels = labels.to(DEVICE) # zero the parameter gradients optimizer.zero_grad() # forward # track history if only in train with torch.set_grad_enabled(phase == 'train'): outputs = model(inputs) _, preds = torch.max(outputs, 1) loss = criterion(outputs, labels) # backward + optimize only if in training phase if phase == 'train': loss.backward() optimizer.step() # scaler.scale(loss).backward() # scaler.step(optimizer) # scaler.update() # statistics running_loss += loss.item() * inputs.size(0) running_corrects += torch.sum(preds == labels.data) print("running loss ", running_loss) if phase == 'train': scheduler.step() epoch_loss = running_loss / dataset_sizes[phase] epoch_acc = running_corrects.double() / dataset_sizes[phase] loss_p[phase].append(epoch_loss) acc_p[phase].append(epoch_acc) print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc)) # deep copy the model if phase == 'val' and epoch_acc > best_acc: best_acc = epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) checkpoint = { 'epoch': epoch, 'valid_acc': best_acc, 'state_dict': model.state_dict(), 'optimizer': optimizer_ft.state_dict(), } checkpoint_path = "/content/drive/MyDrive/competitions/recog-r2/rses50_tts_2_albu.pt" save_ckp(checkpoint, checkpoint_path) print() time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val Acc: {:4f}'.format(best_acc)) # load best model weights model.load_state_dict(best_model_wts) plot(loss_p, acc_p, num_epochs) return model, best_acc
pi # In[5]: pi[0] = [0.75, 0.11, 0.14] for s, pi_s in enumerate(pi): for a, p in enumerate(pi_s): print(f'pi(A={a}|S={s}) = {p.round(2)} ', end='') print() # In[6]: V[0] = 1 tools.plot(V, pi) # We can visualize a value function and policy with the `plot` function in the `tools` module. On the left, the value function is displayed as a barplot. State zero has an expected return of ten, while the other states have an expected return of zero. On the right, the policy is displayed on a two-dimensional grid. Each vertical strip gives the policy at the labeled state. In state zero, action zero is the darkest because the agent's policy makes this choice with the highest probability. In the other states the agent has the equiprobable policy, so the vertical strips are colored uniformly. # You can access the state space and the action set as attributes of the environment. # In[7]: env.S # In[8]: env.A # You will need to use the environment's `transitions` method to complete this assignment. The method takes a state and an action and returns a 2-dimensional array, where the entry at $(i, 0)$ is the reward for transitioning to state $i$ from the current state and the entry at $(i, 1)$ is the conditional probability of transitioning to state $i$ given the current state and action.
def plotVars(var, particle, eras, runMap, varMap, verbose): var_label = varMap[var]["label"] h_var = varMap[var]["h_var"] histogramsData = [] histogramsMC = [] histogramsRatio = [] labelsData = [] labelsMC = [] labelsRatio = [] for era in eras: runDir = runMap[era] result_file = "condor/" + runDir + "/result.root" if verbose: print "{0}: {1}".format(era, result_file) # read input file f = ROOT.TFile(result_file, "read") # --- Data --- # h_name = "{0}/DataMC_{1}_Baseline_{2}_jetpt30{0}{0}Datadata".format(h_var, particle, var) h_data = f.Get(h_name) # use deepcopy so that histogram exists after file is closed / reassigned histogramsData.append(copy.deepcopy(h_data)) labelsData.append(era + " Data") if not histogramsData[-1]: print "ERROR: Unable to load histogram {0}".format(h_name) # --- MC --- # h1_name = "{0}/DataMC_{1}_Baseline_{2}_jetpt30{0}{0}DYstack".format(h_var, particle, var) h2_name = "{0}/DataMC_{1}_Baseline_{2}_jetpt30{0}{0}t#bar{{t}}stack".format(h_var, particle, var) h3_name = "{0}/DataMC_{1}_Baseline_{2}_jetpt30{0}{0}Single tstack".format(h_var, particle, var) h4_name = "{0}/DataMC_{1}_Baseline_{2}_jetpt30{0}{0}Rarestack".format(h_var, particle, var) h1 = f.Get(h1_name) h2 = f.Get(h2_name) h3 = f.Get(h3_name) h4 = f.Get(h4_name) h_mc = h1.Clone("h_mc") h_mc.Add(h2) h_mc.Add(h3) h_mc.Add(h4) # use deepcopy so that histogram exists after file is closed / reassigned histogramsMC.append(copy.deepcopy(h_mc)) labelsMC.append(era + " MC") if not histogramsMC[-1]: print "ERROR: Unable to load histogram {0}".format(h_name) # --- Ratio --- # h_ratio = h_data.Clone("h_ratio") h_ratio.Divide(h_mc) # use deepcopy so that histogram exists after file is closed / reassigned histogramsRatio.append(copy.deepcopy(h_ratio)) labelsRatio.append(era + " Data/MC") if not histogramsRatio[-1]: print "ERROR: Unable to load histogram {0}".format(h_name) nameData = "{0}_data_{1}".format(particle, var_label) titleData = "{0} data comparison for {1}".format(particle, var_label) nameMC = "{0}_mc_{1}".format(particle, var_label) titleMC = "{0} MC comparison for {1}".format(particle, var_label) nameRatio = "{0}_ratio_{1}".format(particle, var_label) titleRatio = "{0} Data/MC comparison for {1}".format(particle, var_label) x_title = var_label x_min = 0 x_max = 11 # standard y-axis limits #y_min_1 = 10**-2 #y_max_1 = 10**6 # normalized y-axis limits y_min_1 = 10**-7 y_max_1 = 10**1 y_min_2 = 0.0 y_max_2 = 3.0 # plot(histograms, labels, name, title, x_title, x_min, x_max, y_min, y_max, era, showStats=False, normalize=False, setLog=False) plot(histogramsData, labelsData, nameData, titleData, x_title, x_min, x_max, y_min_1, y_max_1, "Run2", showStats=False, normalize=True, setLog=True) plot(histogramsMC, labelsMC, nameMC, titleMC, x_title, x_min, x_max, y_min_1, y_max_1, "Run2", showStats=False, normalize=True, setLog=True) plot(histogramsRatio, labelsRatio, nameRatio, titleRatio, x_title, x_min, x_max, y_min_2, y_max_2, "Run2", showStats=False, normalize=False, setLog=False)