def readGeo(self): ''' Read out the *_geom.tab file, in which the column are: col, time, lat, lon, mrad, srad, rvel, tvel, sza, phase ''' self.geoinfo = [] geofile = self.datapath + '/s_' + self.trackID + '_geom.tab' try: with open(geofile,'r') as f: for line in f: line = line.replace('\r\n','').split(',') # read lat, lon, Hmars, Hsat self.geoinfo.append([float(line[2]),float(line[3]),float(line[4])*1000,float(line[5])*1000]) self.geoinfo = np.asarray(self.geoinfo) # delay time between satellite and the Mars aeroid time = 2 * (self.geoinfo[:,3] - self.geoinfo[:,2]) /self.C time = time.reshape((len(time),1)) self.geoinfo = np.concatenate((self.geoinfo,time),axis = 1) # keep lat and lon to the end of self.geoinfo # before replacing them to mapx, mapy #self.geoinfo = np.concatenate((self.geoinfo, self.geoinfo[:,:2]), axis = 1) # transform from Mars 2000 lat/lon to MOLA sphere projection map coordinate x/y ptsx,ptsy = pj.transform(self.mars2000ll, self.molaeqc, self.geoinfo[:,1],self.geoinfo[:,0]) self.geoinfo[:,0] = ptsx self.geoinfo[:,1] = ptsy except: self.geoinfo = None
def get_era_winds(m, rawdatapath, start_year, end_year, start_month, end_month, xyres): wind_data = Dataset(rawdatapath+'/WINDS/ERA/DATA/ERAI_WINDS_MONTHLY_1979-2014.nc', 'r') lats = wind_data.variables['latitude'][::-1] lons = wind_data.variables['longitude'][:] time = wind_data.variables['time'][:]/(24.*30.) time = time-time[0] time=time.reshape(time.shape[0]/12,12) u10 = wind_data.variables['u10'][:, ::-1, :] v10 = wind_data.variables['v10'][:, ::-1, :] u10=u10.reshape(u10.shape[0]/12, 12,u10.shape[1],u10.shape[2]) v10=v10.reshape(v10.shape[0]/12, 12, v10.shape[1],v10.shape[2]) u10_winter_mean= np.mean(u10[start_year-1979:end_year-1979+1,start_month:end_month+1], axis=tuple(range(0, 2))) v10_winter_mean= np.mean(v10[start_year-1979:end_year-1979+1,start_month:end_month+1], axis=tuple(range(0, 2))) u10_winter_meanS, lonsS = shiftgrid(180.,u10_winter_mean,lons,start=False) v10_winter_meanS, lonsS = shiftgrid(180.,v10_winter_mean,lons,start=False) u10_winter_meanSC, lonsSC = addcyclic(u10_winter_meanS, lonsS) v10_winter_meanSC, lonsSC = addcyclic(v10_winter_meanS, lonsS) xyres=100 xvel,yvel,xptsW,yptsW = m.transform_vector(u10_winter_meanSC,v10_winter_meanSC,lonsSC,lats,xyres,xyres,returnxy=True,masked=True) wind_speed = sqrt((xvel**2) + (yvel**2)) wind_speed = sqrt((xvel**2) + (yvel**2)) return xptsW, yptsW, xvel, yvel, wind_speed
def propagateSignal(sig, time, fs, freq=None, tone=None): # to handle single scalar time shift if not isinstance(time, np.ndarray): time = np.array([time]) # to handle 1-D input if sig.ndim == 1: sig = sig.reshape( (1, -1)) # automatic 2-d row vector detection using -1 # generate a tone if no tone is passed in and a freqshift is desired if freq is not None and tone is None: print('Generating tone for freq shift..') tone = np.exp(1j * 2 * np.pi * freq * np.arange(sig.shape[1]) / fs) # propagate the signal in time sigfft = np.fft.fft(sig) # this automatically ffts each row sigFreq = makeFreq(sig.shape[1], fs).reshape( (1, sig.shape[1])) # construct 2-d, row vector mat = np.exp( 1j * 2 * np.pi * sigFreq * -time.reshape((len(time), 1)) ) # construct 2d matrix for each row having its own time shift preifft = mat * sigfft result = np.fft.ifft(preifft) # no freq shift, just return if tone is None: return result # otherwise return the freqshifted version with the tone else: print('Returning shifted signal + tone used.') return result * tone, tone
def rolling_max_slope(data, base=0.00006, action='buy'): if action == 'buy': is_buy = True is_sell = False elif action == 'sell': is_buy = False is_sell = True l_ = len(data) tmp = data[:, :, 0] if is_buy: tmp1 = torch.zeros(tmp.shape) elif is_sell: tmp1 = torch.ones(tmp.shape)*100 ret = torch.cat((tmp, tmp1), 0) ret = ret.unfold(0, l_, 1) ret = ret[:-1] print(ret.shape) if is_buy: diff = ret[1:] - ret[0] - base if is_sell: diff = ret[0] - ret[1:] - base time = torch.range(1, len(diff)) time = time.reshape(-1, 1, 1) slope = diff/time.float() max_slope, index = torch.max(slope, dim=0) time = index.float()+1 max_slope = max_slope/time max_slope = max_slope.permute(1, 0) max_slope = max_slope.unsqueeze(2) data = torch.cat((data, max_slope), 2) return data
def _prepare_input(self): if self.uinput is None: uinput = 0.0 else: time = numpy.r_[0.0:self.simulation_length:self.integrator.dt] self.stimulus.configure_time(time.reshape((1, -1))) uinput = numpy.zeros((self.model.nvar, self.number_of_nodes, 1)) return uinput
def run_epoch(session, m, data, time, eval_op, state=None): """Runs the model on the given data.""" x = data.reshape((1, 1, input_dim+types)) t = time.reshape((1, 1, 1)) prob, _state, _ = session.run([m._prob, m.final_state, eval_op], {m.input_data: x, # m.input_time: t, m.initial_state: state}) return prob, _state
def _prepare_stimulus(self): if self.stimulus is None: stimulus = 0.0 else: time = numpy.r_[0.0:self.simulation_length:self.integrator.dt] self.stimulus.configure_time(time.reshape((1, -1))) stimulus = numpy.zeros((self.model.nvar, self.number_of_nodes, 1)) self.log.debug("stimulus shape is: %s", stimulus.shape) return stimulus
def get_ksens_at_res_time(self, ksens, time_array, res_time): ''' Helper function that takes the full time history of kinetic sensitivities and returns the data at the time step for which the residence time occurs. Using linear interpolation if needed. Parameters ---------- ksens : numpy array Three dimensional numpy array that contains kinetic sensitivities. time_array : pandas series Time column of time history pandas data frame. res_time : float Residence time value. Returns ------- ksens_array : numpy array kinetic sensitivity array where all times but the residence time have been removed. temp_arrays : numpy array Variable for testing. ''' ksens_array = [] temp_arrays = [] for sheet in range(ksens.shape[2]): temp = ksens[:, :, sheet] time = time_array.values time = time.reshape((time.shape[0], 1)) temp_with_time = np.hstack((time, temp)) df = copy.deepcopy(temp_with_time) df = pd.DataFrame(temp_with_time) df = df.rename(columns={0: 'time'}) temp_arrays.append(df) df.loc[-1, 'time'] = float(res_time) df = df.sort_values('time').reset_index(drop=True) df = df.interpolate() res_time_k_sens_data = df.iloc[(df['time'] - res_time).abs().argsort()[:1]] res_time_k_sens_data = res_time_k_sens_data.reset_index(drop=True) res_time_k_sens_data = res_time_k_sens_data.drop(columns="time") res_time_k_sens_data = res_time_k_sens_data.to_numpy() res_time_k_sens_data = res_time_k_sens_data.reshape( (res_time_k_sens_data.shape[0], res_time_k_sens_data.shape[1], 1)) ksens_array.append(res_time_k_sens_data) ksens_array = np.dstack((ksens_array)) return ksens_array, temp_arrays
def parse_datafiles(targetfile, binno, outdir): item = targetfile # for item in filelist: f = open(item, 'r') a = f.readlines() binnumber = 1024 counter = 0 spectra = np.zeros((0, binnumber)) timetracker = 0 energy_deposited = [] for i in range(len(a)): b = a[i].strip() b_parsed = b.split(',') event_time = int(b_parsed[0]) energy_deposited += [float(b_parsed[1])] timetracker += event_time # print(timetracker) if timetracker >= 1E6: timetracker = 0 source_id = 0 counts, energy_edges = np.histogram(energy_deposited, bins=binnumber, range=(0.0, 3000.0)) spectra = np.vstack((spectra, counts)) counter += 1 energy_deposited = [] # if counter >= 100: # break # print(np.sum(spectra[0, :])) time = np.linspace(0, counter, counter) time = time.reshape((time.shape[0], 1)) # print(time.shape, spectra.shape) tosave = np.hstack((time, spectra)) f.close() head, tail = os.path.split(item) print(tail, spectra.shape) # f = open(os.path.join('./integrations', tail), 'w') # np.savetxt(f, tosave, delimiter=',') # f.close() np.save(os.path.join(outdir, tail[:-4] + '.npy'), tosave) return
def full_benchmark(num_env, num_traj, experiment_type, save=True, config='default', report=True, params=None, system='acrobot_obs', traj_id_offset=1800): sr = np.zeros((num_env, num_traj)) time = np.zeros((num_env, num_traj)) costs = np.zeros((num_env, num_traj)) for env_id in range(num_env): for traj_id in range(num_traj): experiment_func = importlib.import_module( ".{}_exp".format(experiment_type), package="experiments").experiment result = experiment_func(env_id, traj_id + traj_id_offset, params=params, system=system) sr[env_id, traj_id] = result['successful'] if result['successful']: time[env_id, traj_id] = result['planning_time'] costs[env_id, traj_id] = result['costs'] if save: Path("results/cpp_full/{}/{}/".format(system, config)).mkdir( parents=True, exist_ok=True) np.save( 'results/cpp_full/{}/{}/sr_{}_{}.npy'.format( system, config, num_env, num_traj), sr) np.save( 'results/cpp_full/{}/{}/time_{}_{}.npy'.format( system, config, num_env, num_traj), time) np.save( 'results/cpp_full/{}/{}/costs_{}_{}.npy'.format( system, config, num_env, num_traj), costs) if report: sr_list = sr.reshape(-1)[:(num_traj * env_id + traj_id + 1)] mask = sr_list > 0 print("sr:{}\ttime:{}\tcosts:{}".format( sr_list.mean(), time.reshape(-1)[:(num_traj * env_id + traj_id + 1)][mask].mean(), costs.reshape(-1)[:(num_traj * env_id + traj_id + 1)][mask].mean()))
def full_benchmark(num_env, num_traj, save=True, config='default', report=True, params_module=None, system='acrobot_obs'): sr = np.zeros((num_env, num_traj)) time = np.zeros((num_env, num_traj)) costs = np.zeros((num_env, num_traj)) for env_id in range(num_env): for traj_id in range(num_traj): result = experiment(env_id, traj_id, params_module=params_module, system=system) sr[env_id, traj_id] = result['successful'] if result['successful']: time[env_id, traj_id] = result['planning_time'] costs[env_id, traj_id] = result['costs'] if save: Path("results/cpp_full/{}/{}/".format(config, system)).mkdir( parents=True, exist_ok=True) np.save( 'results/cpp_full/{}/{}/sr_{}_{}.npy'.format( config, system, num_env, num_traj), sr) np.save( 'results/cpp_full/{}/{}/time_{}_{}.npy'.format( config, system, num_env, num_traj), time) np.save( 'results/cpp_full/{}/{}/costs_{}_{}.npy'.format( config, system, num_env, num_traj), costs) if report: print("sr:{}\ttime:{}\tcosts:{}".format( sr.reshape(-1)[:(num_traj * env_id + traj_id + 1)].mean(), time.reshape(-1)[:(num_traj * env_id + traj_id + 1)].mean(), costs.reshape(-1)[:(num_traj * env_id + traj_id + 1)].mean(), ))
def mag(xyz: np.array) -> np.array: time = xyz[:, 0] L2_mag = (xyz[:, -3]**2 + xyz[:, -2]**2 + xyz[:, -1]**2)**(0.5) return np.hstack((time.reshape(-1, 1), L2_mag.reshape(-1, 1)))
def __init__(self, data, delta, lambd, nonsnps=0, time=None, seed=16962, sigma='power', TType=torch.DoubleTensor): """ data: a Tensor. delta: indicator for right censoring (1 if uncensored, 0 if censored) lambd: regularization parameter """ self.rank = dist.get_rank() self.size = dist.get_world_size() self.seed = seed torch.manual_seed(seed) self.TType = TType self.n, self.p = data.shape n, p = self.n, self.p print(n, p) self.data = data.type(TType) self.delta = delta.type(TType) self.delta_dist = distmat.dist_data(self.delta, TType=TType) self.prev_obj = -inf self.beta = distmat.dist_data(torch.zeros((p, 1)).type(TType), TType=TType) self.beta_prev = distmat.dist_data(torch.zeros((p, 1)).type(TType), TType=TType) self.datat = self.data.t() if sigma == 'power': l2 = self.power() self.sigma = (1 / (2 * (l2**2))).item() elif sigma == 'quicknorm': b = self.spectral_upper_bdd() self.sigma = (1 / (2 * (b**2))).item() else: self.sigma = sigma print(self.sigma) self.lambd = lambd self.nonsnps = nonsnps self.soft_threshold_ftn = torch.nn.Softshrink(lambd) if time is None: time = -torch.arange(0, n).view(-1, 1) self.breslow_ind = torch.tensor(breslow_ind(time.cpu().numpy())).to( dtype=torch.int64, device=self.beta.chunk.device) time_local = time.reshape(-1, 1).type(TType) time_dist = distmat.dist_data(time, TType=self.TType) #r_local = torch.arange(0, n).view(-1, 1).type(TType) #r_dist = distmat.dist_data(r_local, TType=self.TType) self.pi_ind = (time_dist - time_local.t() >= 0).type(TType)
ddq1 = (A[0, 0] * dq1 + A[0, 1] * dq2 + Z[0] + D2[0, 0] * (-Ta[0, 0] + tol1) + D2[0, 1] * (-Ta[1, 0] + tol2))[0] ddq2 = (A[1, 0] * dq1 + A[1, 1] * dq2 + Z[1] + D2[1, 0] * (-Ta[0, 0] + tol1) + D2[1, 1] * (-Ta[1, 0] + tol2))[0] dq1 = dq1 + ddq1 * dts dq2 = dq2 + ddq2 * dts q1 = q1 + dq1 * dts q2 = q2 + dq2 * dts q[t, :] = [q1, q2] dq[t, :] = [dq1, dq2] pass # q_ex.append(q) # dq_ex.append(dq) # ax = fig.add_subplot(2, 1, 1) e_ex[i, :] = np.mean(e, axis=0) ax1[0].plot(time.reshape(time.shape[0], 1), q[:, 0], color=seaborn.xkcd_rgb[color_map[i]]) ax1[0].plot(time.reshape(time.shape[0], 1), q1d, color='k', linestyle='--') # ax = fig.add_subplot(2, 1, 2) ax1[1].plot(time.reshape(time.shape[0], 1), q[:, 1], color=seaborn.xkcd_rgb[color_map[i]]) ax1[1].plot(time.reshape(time.shape[0], 1), q2d, color='k', linestyle='--') fig1.canvas.flush_events()
def parse_datafiles(targetfile, binno): # energy = np.linspace(0, 3000, 1024) # filelist = ['/home/holiestcow/Documents/zephyr/datasets/muse/trainingData/109798.csv'] # process all files. # filelist = glob.glob('/home/holiestcow/Documents/zephyr/datasets/muse/trainingData/*.csv') item = targetfile # for item in filelist: f = open(item, 'r') a = f.readlines() binnumber = 1024 counter = 0 spectra = np.zeros((0, binnumber)) timetracker = 0 energy_deposited = [] for i in range(len(a)): b = a[i].strip() b_parsed = b.split(',') event_time = int(b_parsed[0]) energy_deposited += [float(b_parsed[1])] timetracker += event_time # print(timetracker) if timetracker >= 1E6: timetracker = 0 source_id = 0 counts, energy_edges = np.histogram(energy_deposited, bins=binnumber, range=(0.0, 4000.0)) spectra = np.vstack((spectra, counts)) counter += 1 # if counter >= 100: # break # print(np.sum(spectra[0, :])) time = np.linspace(0, counter, counter) time = time.reshape((time.shape[0], 1)) # print(time.shape, spectra.shape) tosave = np.hstack((time, spectra)) f.close() head, tail = os.path.split(item) print(tail, spectra.shape) # f = open(os.path.join('./integrations', tail), 'w') # np.savetxt(f, tosave, delimiter=',') # f.close() np.save(os.path.join('./integrations', tail[:-4] + '.npy'), tosave) # features = nscrad_rebin(spectra, 64) # features_bg = features[:30, :] # nuisance = nscrad_get_nuisance() # # energy = energy_edges[1:] # nuisance_matrix = np.zeros((0, binno)) # for key in nuisance: # fig = plt.figure() # plt.plot(nuisance[key]['energy'], nuisance[key]['counts']) # fig.savefig('nuisance_{}.png'.format(key)) # print([0] + nuisance[key]['energy']) # print(len(nuisance[key]['energy']), len(nuisance[key]['counts'])) # # out = np.array(rebin(np.array([0] + nuisance[key]['energy']), np.array(nuisance[key]['counts']), # energy_edges)) # out = out.reshape((1, len(out))) # nuisance_matrix = np.vstack((nuisance_matrix, out)) # # nuisance_matrix = nscrad_rebin(nuisance_matrix, 64) # nuisance_matrix = nuisance_matrix.reshape((nuisance_matrix.shape[1], nuisance_matrix.shape[0])) # bullshit = nscrad_get_parameters(features, features_bg, nuisance_matrix) return
def fit(self, data_set, test_set): self._sess.run(tf.global_variables_initializer()) data_set.epoch_completed = 0 for c in tf.trainable_variables(self._name): print(c.name) print("acc\tauc\tepoch\tloss\tloss_diff\tcount") logged = set() loss = 0 count = 0 while data_set.epoch_completed < self._epochs: dynamic_features, time, labels = data_set.next_batch( self._batch_size) self._sess.run(self._train_op, feed_dict={ self._x: dynamic_features, self._y: labels, self._t: time.reshape(-1, dynamic_features.shape[1], 1) }) # self._sess.run(self._train_op, # feed_dict={self._x: dynamic_features[:, :, 1:], # self._y: labels}) if data_set.epoch_completed % self._output_n_epoch == 0 and data_set.epoch_completed not in logged: logged.add(data_set.epoch_completed) loss_prev = loss loss = self._sess.run(self._loss, feed_dict={ self._x: data_set.dynamic_features, self._y: data_set.labels, self._t: data_set.time.reshape( -1, dynamic_features.shape[1], 1) }) # loss = self._sess.run(self._loss, # feed_dict={self._x: data_set.dynamic_features[:, :,1:].reshape(-1, self._time_steps,self._num_features), # self._y: data_set.labels}) loss_diff = loss_prev - loss y_score = self.predict(test_set) y_score = y_score.reshape([ -1, ]) test_labels = test_set.labels test_labels = test_labels.reshape([ -1, ]) auc = roc_auc_score(test_labels, y_score) print("{}\t{}\t{}\t{}\t{}\t{}".format( data_set.epoch_completed, auc, loss, loss_diff, count, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) # 设置训练停止条件 if loss > self._max_loss: count = 0 else: if loss_diff > self._max_pace: count = 0 else: count += 1 if count > 9: break # save_path = self._save.save(self._sess, self._name + "model/save_net" +time.strftime("%m-%d-%H-%M-%S", time.localtime()) + ".ckpt") # t = time.localtime() t = datetime.datetime.now().strftime("%m-%d-%H-%M-%S") save_path = self._save.save( self._sess, self._name + "model/save_net" + t) + ".ckpt" print("Save to path: ", save_path)
'd3_dot': str(joint_vel[2, i]) }) rate.sleep() print('Export Complete.') print('--------------------------------------') fig, axs = plt.subplots(2, 2) fig.suptitle('Kinematic Trajectory Data') axs[0, 0].plot(planned_time.reshape(N, 1), joint_pos[0, :].reshape(N, 1), planned_time.reshape(N, 1), joint_pos[1, :].reshape(N, 1), planned_time.reshape(N, 1), joint_pos[2, :].reshape(N, 1)) axs[0, 0].set_title('Planned Joint Positions') axs[0, 1].plot(time.reshape(data_points, 1), measured_joint_pos[0, :].reshape(data_points, 1), time.reshape(data_points, 1), measured_joint_pos[1, :].reshape(data_points, 1), time.reshape(data_points, 1), measured_joint_pos[2, :].reshape(data_points, 1)) axs[0, 1].set_title('Measured Joint Positions') axs[1, 0].plot(time.reshape(data_points, 1), measured_joint_eff[0, :].reshape(data_points, 1), time.reshape(data_points, 1), measured_joint_eff[1, :].reshape(data_points, 1), time.reshape(data_points, 1), measured_joint_eff[2, :].reshape(data_points, 1)) axs[1, 0].set_title('Measured Joint Efforts') plt.show()