def __init__(self): with open(os.path.join(os.path.dirname(__file__), 'Config.json'), 'r') as json_file: config = json.load(json_file) self._shots = dict() self._shots['train'] = config['shots']['train'] self._shots['test'] = config['shots']['test'] self._shots['val'] = config['shots']['test'] self._tags = config['diagnosis']['tags'] # 截取信号的起始时间(ms) self._start_time = config['diagnosis']['start_time'] # 重采样的信号频率(kHz) self._sample_rate = config['diagnosis']['sample_rate'] self._normalized = config['normalized'] self._directory = config['directory'] if not os.path.exists(self._directory): os.makedirs(self._directory) # 复制配置文件到输出路径 with open(os.path.join(self._directory, 'Config.json'), 'w') as json_file: json.dump(config, json_file, indent=4) exit() ddb = Query() if self._normalized: self._normalize_param = ddb.get_normalize_parm(self._tags)
def __init__(self, normalized=False): config = configparser.ConfigParser() config.read( os.path.join(os.path.dirname(__file__), 'DataSetConfig.ini')) self._tags = ast.literal_eval(config['Diagnosis']['tags']) self._sample_rate = int(config['Diagnosis']['sample_rate']) self._frame_size = int(config['Diagnosis']['frame_size']) self._step = int(config['Diagnosis']['step']) self._npy_path = config['path']['npy'] if not os.path.exists(self._npy_path): os.makedirs(self._npy_path) ddb = Query() self._normalized = normalized if normalized: self._normalize_param = ddb.get_normalize_parm(self._tags) my_query = {'IsValidShot': True, 'IsDisrupt': False} self._shots = ddb.query(my_query) my_query = { 'IsValidShot': True, 'IsDisrupt': True, 'CqTime': { "$gte": 0.05 }, 'IpFlat': { '$gte': 110 } } self._shots += ddb.query(my_query)
def run(self): data_reader = Reader() ddb = Query() for shot in self._shots: # if shot < 1065500 or shot > 1065599: # continue print(shot) try: tags = ddb.tag(shot) if tags['IsDisrupt']: t1 = tags['CqTime'] else: t1 = tags['RampDownTime'] new_dig_length = int((t1 * 1000 - 50) * self._sample_rate) data = data_reader.read_many(shot, self._tags) digs = [] for tag, (dig, time) in data.items(): dig = dig[(0.05 <= time) & (time <= t1)] if self._normalized: dig = (dig - self._normalize_param[tag]['min']) / \ (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min']) digs.append(signal.resample(dig, new_dig_length)) digs = np.array(digs) y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt']) index = 0 path = os.path.join(self._npy_path, '{}'.format(shot)) if not os.path.exists(path): os.makedirs(path) while index + self._frame_size <= new_dig_length: frame = digs[:, index:index + self._frame_size] y_frame = y_[index:index + self._frame_size] np.save( os.path.join( path, 'x_{}.npy'.format(int(index / self._step))), frame) np.save( os.path.join( path, 'y_{}.npy'.format(int(index / self._step))), y_frame) index += self._step if index + self._frame_size - new_dig_length < self._frame_size / 2: frame = digs[:, new_dig_length - self._frame_size:new_dig_length] y_frame = y_[new_dig_length - self._frame_size:new_dig_length] np.save( os.path.join( path, 'x_{}.npy'.format(int(index / self._step))), frame) np.save( os.path.join( path, 'y_{}.npy'.format(int(index / self._step))), y_frame) except Exception as e: print(e) traceback.print_exc()
def get(self): """ 加载npy数据到tf.data.DataSet :return: training set, test set """ train_test_shots = list() with open(os.path.join('log', 'ShotsUsed4Training.txt'), 'r') as f: for shot in f.readlines(): train_test_shots.append(int(shot)) train_test_shots.sort(reverse=False) ddb = Query() shots = list() with open(os.path.join('log', 'ShotsInDataset.txt'), 'w') as f: my_query = { 'IsValidShot': True, 'IsDisrupt': True, 'CqTime': { "$gte": 0.15 }, 'IpFlat': { '$gte': 110 } } for shot in ddb.query(my_query): if os.path.exists( os.path.join(self.npy_path, '{}'.format(shot))): shots.append(shot) shots.sort(reverse=False) for shot in shots: if shot in train_test_shots: print('{} 1 d'.format(shot), file=f) else: print('{} 0 d'.format(shot), file=f) shots.clear() my_query = { 'IsValidShot': True, 'IsDisrupt': False, 'IpFlat': { '$gte': 110 } } for shot in ddb.query(my_query): if os.path.exists( os.path.join(self.npy_path, '{}'.format(shot))): shots.append(shot) shots.sort(reverse=False) for shot in shots: print('{} 0 u'.format(shot), file=f)
def save_full_npy(self, shots): data_reader = Reader() ddb = Query() path = os.path.join(self._npy_path, 'full') if not os.path.exists(path): os.makedirs(path) print('####Start generate val DataSet####') for shot in shots: try: print(shot) tags = ddb.tag(shot) if tags['IsDisrupt']: t1 = tags['CqTime'] else: t1 = tags['RampDownTime'] new_dig_length = int((t1 * 1000 - 50) * self._sample_rate) data = data_reader.read_many(shot, self._tags) digs = [] for tag, (dig, time) in data.items(): dig = dig[(0.05 <= time) & (time <= t1)] if self._normalized: dig = (dig - self._normalize_param[tag]['min']) / \ (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min']) digs.append(signal.resample(dig, new_dig_length)) digs = np.array(digs) y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt']) index = 0 x = list() labels = list() while index + self._frame_size <= new_dig_length: frame = digs[:, index:index + self._frame_size] y_frame = y_[index:index + self._frame_size] # index += self.frame_size x.append(frame) labels.append(y_frame[-1]) index += self._step x = np.array(x) labels = np.array(labels) np.save(os.path.join(path, 'x_{}.npy'.format(shot)), x) np.save(os.path.join(path, 'y_{}.npy'.format(shot)), labels) except Exception as e: print(e) traceback.print_exc()
def generate(self): data_reader = Reader() ddb = Query() for categories, shots in self._shots.items(): if not os.path.exists(os.path.join(self._directory, categories)): os.makedirs(os.path.join(self._directory, categories)) for shot in shots: print(shot) try: tags = ddb.tag(shot) if tags['IsDisrupt']: t1 = tags['CqTime'] else: t1 = tags['RampDownTime'] new_dig_length = int( (t1 * 1000 - self._start_time) * self._sample_rate) data = data_reader.read_many(shot, self._tags) digs = [] for tag, (dig, time) in data.items(): dig = dig[(self._start_time / 1000 <= time) & (time <= t1)] # 归一化 if self._normalized: dig = (dig - self._normalize_param[tag]['min']) / \ (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min']) # 重采样 digs.append(signal.resample(dig, new_dig_length)) digs = np.array(digs) f = h5py.File( os.path.join(self._directory, categories, '{}.hdf5'.format(shot))) dataset = f.create_dataset('diagnosis', data=digs) for key, value in tags.items(): dataset.attrs.create(key, value) f.close() except Exception as e: print(e) traceback.print_exc()
def plot_much(self, Taglist=None, Shotlist=None, Savepath=None, ShowDownTime=False, ShowIpFlat=False, xline=None, yline=None): if Savepath: root_path = Savepath if not os.path.exists(root_path): raise ValueError( 'No such saving path, you need to create one! ') else: root_path = os.getcwd() + os.sep + "plot" print(root_path) if not os.path.exists(root_path): os.makedirs(root_path) for tag in Taglist: tag_name = tag[1:] file_path = root_path + os.sep + tag_name if not os.path.exists(file_path): os.makedirs(file_path) reader = Reader(root_path=self.hdf5path) db = Query() for tag in Taglist: tag_name = tag[1:] file_path = root_path + os.sep + tag_name n = 1 for shot in Shotlist: print("Shot:{}".format(shot) + " Tag:{} ".format(tag_name) + "No.{}".format(n)) n += 1 try: shot_info = db.tag(int(shot)) data = reader.read_one(int(shot), tag) plt.figure((str(shot) + tag_name)) plt.plot(data[1], data[0], 'g') if ShowDownTime: if shot_info["IsValidShot"]: if shot_info["IsDisrupt"]: plt.axvline(round(shot_info["CqTime"], 3), c='r') else: plt.axvline(round(shot_info["RampDownTime"], 3), c='r') if ShowIpFlat: if tag == r"\ip": if shot_info["IsValidShot"]: plt.axhline(round(shot_info["IpFlat"], 3), c='k') if xline: if not isNum(xline): raise ValueError('xline needs to be number ') plt.axvline(round(xline, 3)) if yline: if not isNum(yline): raise ValueError('yline needs to be number ') plt.axhline(round(yline, 3)) path = file_path + os.sep + r"{}.png".format(shot) plt.savefig(path) plt.close() except Exception as err: print("Shot:{}".format(shot) + " Tag:{} ".format(tag_name) + "No data") plt.close() pass
import os import numpy as np from DDB.Service import Query # IsValidShot db = Query() valid_query = { 'IsValidShot': True, 'IpFlat': { '$gt': 100 }, "$or": [{ 'CqTime': { '$gt': 0.2 } }, { 'RampDownTime': { '$gt': 0.2 } }] } break_query = { 'IsValidShot': True, 'IsDisrupt': True, 'IpFlat': { '$gt': 100 }, 'CqTime': { '$gt': 0.2 } }
import DDB from pymongo import MongoClient from DDB.Service import Query from DDB.Data import Reader import random from scipy import signal ddb = Query() my_query = {'IsValidShot': True, 'IsDisrupt': False} shots = ddb.query(my_query) my_query = { 'IsValidShot': True, 'IsDisrupt': True, 'CqTime': { "$gte": 0.05 }, 'IpFlat': { '$gte': 110 } } shots += ddb.query(my_query) config = DDB.get_config() database = config['output'] client = MongoClient(database['host'], int(database['port'])) db = client[database['database']] param = db[database['collection'] + '归一化参数'] # tags = [r'\Bt', r'\Ihfp', r'\Ivfp', r'\MA_POL_CA01T', r'\MA_POL_CA02T', r'\MA_POL_CA03T', r'\MA_POL_CA05T', r'\MA_POL_CA06T', r'\MA_POL_CA07T', r'\MA_POL_CA19T', r'\MA_POL_CA20T', r'\MA_POL_CA21T', r'\MA_POL_CA22T', r'\MA_POL_CA23T', r'\MA_POL_CA24T', r'\axuv_ca_01', r'\ip', r'\sxr_cb_024', r'\sxr_cc_049', r'\vs_c3_aa001', r'\vs_ha_aa001'] # reader = Reader(root_path='/nas/hdf5_new') # result = dict() # for tag in tags:
def load(self): """ 加载npy数据到tf.data.DataSet :return: training set, test set """ examples_und = list() examples_dis = list() labels_und = list() labels_dis = list() ddb = Query() shots = list() # 不使用非破裂炮进行训练 # my_query = {'IsValidShot': True, 'IsDisrupt': False} # for shot in ddb.query(my_query): # if os.path.exists(os.path.join(self.npy_path, '{}'.format(shot))): # shots.append(shot) # if len(shots) >= self.shots/2: # break my_query = { 'IsValidShot': True, 'IsDisrupt': True, 'CqTime': { "$gte": 0.15 }, 'IpFlat': { '$gte': 110 } } for shot in ddb.query(my_query): if os.path.exists(os.path.join(self._npy_path, '{}'.format(shot))): shots.append(shot) if len(shots) >= self._shots_len: break # shots = np.random.choice(shots, self.shots_len) if not os.path.exists('log'): os.mkdir('log') if os.path.exists(os.path.join('log', 'ShotsUsed4Training.txt')): path = os.path.join( 'log', 'ShotsUsed4Training_{}.txt'.format( time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))) os.rename(os.path.join('log', 'ShotsUsed4Training.txt'), path) with open(os.path.join('log', 'ShotsUsed4Training.txt'), 'w') as f: for shot in shots: print(shot, file=f) for shot in shots: file_names = [ i for i in os.listdir( os.path.join(self._npy_path, '{}'.format(shot))) if 'x' in i ] for file in file_names: x = np.load( os.path.join(self._npy_path, '{}'.format(shot), file)) y = np.load( os.path.join(self._npy_path, '{}'.format(shot), file.replace('x', 'y'))) if y[-1] > 0: examples_dis.append(x) labels_dis.append(y[-1]) else: examples_und.append(x) labels_und.append(y[-1]) len_und = len(labels_und) len_dis = len(labels_dis) print('Length un_disruption: ', len_und, '\nLength disruption: ', len_dis) # -------------------------------------------------------------------------------------- # 均衡策略1:扩大disruption, un_disruption不变 # -------------------------------------------------------------------------------------- # dataset_und = tf.data.Dataset.from_tensor_slices((examples_und, labels_und)) # dataset_dis = tf.data.Dataset.from_tensor_slices((examples_dis, labels_dis)) # # split_point_und = (int(len(labels_und)*self.train), int(len(labels_und)*self.test)) # split_point_dis = (int(len(labels_dis)*self.train), int(len(labels_dis)*self.test)) # # train_dataset_und = dataset_und.take(split_point_und[0]) # test_dataset_und = dataset_und.skip(split_point_und[0]).take(split_point_und[1]) # train_dataset_dis = dataset_dis.take(split_point_dis[0]) # test_dataset_dis = dataset_dis.skip(split_point_dis[0]).take(split_point_dis[1]) # # train_dataset_dis = train_dataset_dis.repeat(int(split_point_und[0]/split_point_dis[0])) # # train_dataset = train_dataset_und.concatenate(train_dataset_dis) # test_dataset = test_dataset_und.concatenate(test_dataset_dis) # -------------------------------------------------------------------------------------- # 均衡策略2:disruption扩大2倍, 随机抽取un_disruption, 比例为und/dis = 6/4 # -------------------------------------------------------------------------------------- dataset_und = tf.data.Dataset.from_tensor_slices( (examples_und, labels_und)) dataset_dis = tf.data.Dataset.from_tensor_slices( (examples_dis, labels_dis)) dataset_und = dataset_und.shuffle(buffer_size=len_und).take(3 * len_dis) dataset_dis = dataset_dis.repeat(2) dataset = dataset_und.concatenate(dataset_dis) dataset = dataset.shuffle(5 * len_dis) train_dataset = dataset.take(int(5 * len_dis * self._train_per)) test_dataset = dataset.skip(int(5 * len_dis * self._train_per)).take( int(5 * len_dis * self._test_per)) return train_dataset, test_dataset
def run(self, data): result = { 'IsLockedMode': False, # 是否发生锁模 'LockedModeTime': 0.0 # 锁模时间 # 'IsUnLockedMode': False, # 是否解锁 # 'UnLockedModeTime': 0.0 # 解锁时间 } shot = data['shot'] db = Query() tag = db.tag(shot) if tag['IsRampUpDisrupt'] is True: return result if tag['IsDisrupt'] is True: end_t = tag['CqTime'] else: end_t = tag['RampDownTime'] # mirnov信号 mirnov_t = data['\MA_POL_CA01T'] if mirnov_t.shape[0] == 2 and mirnov_t.shape[1] != 0: mirnov = mirnov_t[0] time = mirnov_t[1] else: result['NoData'] = True return result # 截取信号 start = np.where(time >= 0.05)[0][0] if time[-1] < end_t: end = len(time) else: end = np.where(time >= end_t)[0][0] mirnov = mirnov[start:end] time = time[start:end] # 信号分析 # 采样率 sampling_rate = 250000 # FFT采样点数 frame_size = 1024 # 低通滤波 # scipy.signal.butter(N, Wn, btype='low', analog=False, output='ba') # N:滤波器的阶数 # Wn:归一化截止频率.计算公式Wn = 2 * 截止频率 / 采样频率 # 滤除50kHz以上的成分,截止频率50000 wn = 2 * 10000 / sampling_rate # plt.figure(0) # plt.subplot(211) # plt.title(u'滤波前') # plt.plot(time, mirnov) [b, a] = signal.butter(3, wn, 'low') mirnov = signal.filtfilt(b, a, mirnov) # plt.subplot(212) # plt.title(u'滤波后') # plt.plot(time, mirnov) # plt.xlabel('time/s') # plt.figure(1, figsize=(19.20, 10.80)) # plt.subplot(311) # plt.plot(time, mirnov) # FFT i = 0 max_sqe = [] max_sqe_t = [] window = np.hanning(frame_size) while i < len(mirnov) - frame_size: frames = mirnov[i:i + frame_size] frames *= window mirnov_fft = np.fft.rfft(frames) / frame_size freqs = np.linspace(0, sampling_rate / 2, int(frame_size / 2 + 1)) mirnov_fft = 20 * np.log10( np.clip(np.abs(mirnov_fft), 1e-20, 1e100)) # plt.figure() # plt.subplot(211) # plt.plot(time[i:i+frameSize], mirnov[i:i + frameSize]) # plt.subplot(212) # plt.plot(freqs, mirnov_fft) # plt.show() max_sqe.append(freqs[np.argmax(mirnov_fft)]) max_sqe_t.append(time[int(i + frame_size / 2)]) i += 100 # 中值滤波,平滑处理 max_sqe = signal.medfilt(max_sqe, 31) # plt.subplot(312) # plt.scatter(max_sqe_t, max_sqe) exsad1_t = data['\exsad1'] if exsad1_t.shape[0] == 2 and exsad1_t.shape[1] != 0: exsad1 = exsad1_t[0] exsad1_time = exsad1_t[1] else: result['NoData'] = True return result exsad7_t = data['\exsad7'] if exsad7_t.shape[0] == 2 and exsad7_t.shape[1] != 0: exsad7 = exsad7_t[0] exsad7_time = exsad7_t[1] else: result['NoData'] = True return result exsad_resample = [] for time in max_sqe_t: index1 = np.where(exsad1_time >= time)[0][0] index2 = np.where(exsad7_time >= time)[0][0] exsad_resample.append(exsad1[index1] * 100 / 2.35 - exsad7[index2] * 100 / 1.79) exsad_resample = signal.medfilt(exsad_resample, 31) for i in range(int(len(max_sqe_t) / 2), len(max_sqe_t)): if max_sqe[i] < 1000 and np.fabs(exsad_resample[i]) > 10: result['IsLockedMode'] = True result['LockedModeTime'] = max_sqe_t[i] # plt.subplot(313) # plt.plot(max_sqe_t, exsad_resample) # # plt.scatter(max_sqe_t, exsad7_resample) # plt.xlabel('time/s') # plt.show() # shot = data['shot'] # plt.savefig('image\\mirnov' + os.sep + '{}.png'.format(shot), dpi=300) # plt.close(1) return result
TrainBreak = np.load(root_path + os.sep + r"TrainData" + os.sep + r"TrainBreak.npy") TrainNormal = list(TrainNormal) TrainBreak = list(TrainBreak) print(len(TrainNormal)) print(len(TrainBreak)) TrainShot = TrainNormal + TrainBreak save_path = root_path + os.sep + r"ReduceSampling" if not os.path.exists(save_path): os.makedirs(save_path) n = 1 mistake = [] reader = Reader() db = Query() for shot in TrainShot: print("Shot:{} ".format(shot) + "No.{}".format(n)) n += 1 try: shot_info = db.tag(int(shot)) file = h5py.File(save_path + os.sep + r"{}.hdf5".format(shot)) if not shot_info["IsDisrupt"]: DownTime = shot_info["RampDownTime"] for shottag in all_tags: dataset = reader.read_one(int(shot), shottag) data = dataset[0] time = dataset[1] data = data[time <= DownTime] time = time[time <= DownTime] data = data[time > 0.2]