예제 #1
0
 def __init__(self, root=None):
     """connect to mdsplus server and init data importer"""
     if root:
         self.exporter = Exporter(root)
         self.reader = Reader(root)
     else:
         self.exporter = Exporter()
         self.reader = Reader()
     # log
     log_dir = os.path.abspath('') + os.sep + 'log'
     if not os.path.exists(log_dir):
         os.makedirs(log_dir)
     self.logger = logging.getLogger(__name__)
     self.logger.setLevel(level=logging.INFO)
     handler = logging.FileHandler(
         log_dir + os.sep + 'JTEXTDataExporter_log_{}.txt'.format(
             time.strftime("%Y_%m_%d %H:%M:%S", time.localtime(
                 time.time()))))
     handler.setLevel(logging.INFO)
     formatter = logging.Formatter(
         '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     handler.setFormatter(formatter)
     console = logging.StreamHandler()
     console.setLevel(logging.INFO)
     self.logger.addHandler(handler)
     self.logger.addHandler(console)
예제 #2
0
 def TCShotlist(self, Taglist=None, Shotlist=None, Detail=False):
     Shotlist = list(Shotlist)
     input = Shotlist.copy()
     reader = Reader(root_path=self.hdf5path)
     n = 1
     IncompleteNumber = []
     for i in Shotlist:
         print("第{}个".format(n))
         n += 1
         ShotTag = reader.tags(int(i))
         for j in Taglist:
             if j in ShotTag:
                 continue
             else:
                 IncompleteNumber.append(i)
                 break
     for i in IncompleteNumber:
         for j in Shotlist:
             if int(i) == int(j):
                 Shotlist.remove(j)
     outlist = Shotlist
     if Detail:
         print("Number of input shots    : {}".format(len(input)))
         print("Number of excluded shots : {}".format(len(IncompleteNumber)))
         print("Number of output shots   : {}".format(len(outlist)))
     return outlist
예제 #3
0
    def run(self):
        reader = Reader(self.hdf5_path)
        for plugin in self.plugins.values():
            signals = plugin.requested_signal()
            for shot in self.shots:
                self.logger.info('Run module {} @shot {}.\n'.format(
                    plugin.__class__.__name__, shot))
                try:
                    data = reader.read_many(shot=shot, tags=signals)
                    result = plugin.generate(data)
                except Exception as e:
                    self.logger.info('An error occurs in module {} @shot {}.\n{}\n{}'.format(
                        plugin.__class__.__name__, shot, e, traceback.format_exc()))
                    result = {}
                if self.output['type'] == "mongodb":
                    from pymongo import MongoClient
                    client = MongoClient(self.output['host'], int(self.output['port']))

                    db = client[self.output['database']]
                    if self.config.has_option('output', 'username'):
                        db.authenticate(self.config.get('output', 'username'), self.config.get('output', 'password'))
                    col = db[self.output['collection']]
                    result['shot'] = shot
                    col.update_one(
                        {"shot": shot},
                        {"$set": result},
                        upsert=True
                    )
                elif self.output['type'] == "stdio":
                    print(result)
                else:
                    raise RuntimeError('配置文件output type选项不正确,可选为:mongodb,stdio')
예제 #4
0
    def run(self):
        data_reader = Reader()
        ddb = Query()
        for shot in self._shots:
            # if shot < 1065500 or shot > 1065599:
            #     continue
            print(shot)
            try:
                tags = ddb.tag(shot)
                if tags['IsDisrupt']:
                    t1 = tags['CqTime']
                else:
                    t1 = tags['RampDownTime']
                new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
                data = data_reader.read_many(shot, self._tags)
                digs = []
                for tag, (dig, time) in data.items():
                    dig = dig[(0.05 <= time) & (time <= t1)]
                    if self._normalized:
                        dig = (dig - self._normalize_param[tag]['min']) / \
                              (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
                    digs.append(signal.resample(dig, new_dig_length))

                digs = np.array(digs)
                y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
                index = 0
                path = os.path.join(self._npy_path, '{}'.format(shot))
                if not os.path.exists(path):
                    os.makedirs(path)
                while index + self._frame_size <= new_dig_length:
                    frame = digs[:, index:index + self._frame_size]
                    y_frame = y_[index:index + self._frame_size]
                    np.save(
                        os.path.join(
                            path, 'x_{}.npy'.format(int(index / self._step))),
                        frame)
                    np.save(
                        os.path.join(
                            path, 'y_{}.npy'.format(int(index / self._step))),
                        y_frame)
                    index += self._step
                if index + self._frame_size - new_dig_length < self._frame_size / 2:
                    frame = digs[:, new_dig_length -
                                 self._frame_size:new_dig_length]
                    y_frame = y_[new_dig_length -
                                 self._frame_size:new_dig_length]
                    np.save(
                        os.path.join(
                            path, 'x_{}.npy'.format(int(index / self._step))),
                        frame)
                    np.save(
                        os.path.join(
                            path, 'y_{}.npy'.format(int(index / self._step))),
                        y_frame)
            except Exception as e:
                print(e)
                traceback.print_exc()
예제 #5
0
    def save_full_npy(self, shots):
        data_reader = Reader()
        ddb = Query()
        path = os.path.join(self._npy_path, 'full')
        if not os.path.exists(path):
            os.makedirs(path)
        print('####Start generate val DataSet####')
        for shot in shots:
            try:
                print(shot)

                tags = ddb.tag(shot)
                if tags['IsDisrupt']:
                    t1 = tags['CqTime']
                else:
                    t1 = tags['RampDownTime']
                new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
                data = data_reader.read_many(shot, self._tags)
                digs = []
                for tag, (dig, time) in data.items():
                    dig = dig[(0.05 <= time) & (time <= t1)]
                    if self._normalized:
                        dig = (dig - self._normalize_param[tag]['min']) / \
                              (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
                    digs.append(signal.resample(dig, new_dig_length))

                digs = np.array(digs)
                y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
                index = 0
                x = list()
                labels = list()
                while index + self._frame_size <= new_dig_length:
                    frame = digs[:, index:index + self._frame_size]
                    y_frame = y_[index:index + self._frame_size]
                    # index += self.frame_size
                    x.append(frame)
                    labels.append(y_frame[-1])
                    index += self._step
                x = np.array(x)
                labels = np.array(labels)
                np.save(os.path.join(path, 'x_{}.npy'.format(shot)), x)
                np.save(os.path.join(path, 'y_{}.npy'.format(shot)), labels)
            except Exception as e:
                print(e)
                traceback.print_exc()
예제 #6
0
파일: Plot.py 프로젝트: axk19970225/DDB
    def plot_one(self,
                 Tag=None,
                 Shot=None,
                 Savepath=None,
                 xline=None,
                 yline=None,
                 Showplot=False):
        if Savepath:
            root_path = Savepath
            if not os.path.exists(root_path):
                raise ValueError(
                    'No such saving path, you need to create one! ')
        else:
            root_path = os.getcwd() + os.sep + "plot"
            print(root_path)
            if not os.path.exists(root_path):
                os.makedirs(root_path)

        reader = Reader(root_path=self.hdf5path)
        try:
            data = reader.read_one(int(Shot), Tag)
            tag_name = Tag[1:]
            plt.figure((str(Shot) + tag_name))
            plt.plot(data[1], data[0], 'g')
            if xline:
                if not isNum(xline):
                    raise ValueError('xline needs to be number ')
                plt.axvline(round(xline, 3))
            if yline:
                if not isNum(yline):
                    raise ValueError('yline needs to be number ')
                plt.axhline(round(yline, 3))
            name = str(Shot) + r" " + tag_name
            path = root_path + os.sep + r"{}.png".format(name)
            plt.savefig(path)
            if Showplot:
                plt.show()
            plt.close()
        except Exception as err:
            print("Shot:{}".format(shot) + " Tag:{}  ".format(tag_name) +
                  "No data")
            plt.close()
            pass
예제 #7
0
    def generate(self):
        data_reader = Reader()
        ddb = Query()
        for categories, shots in self._shots.items():
            if not os.path.exists(os.path.join(self._directory, categories)):
                os.makedirs(os.path.join(self._directory, categories))
            for shot in shots:
                print(shot)
                try:
                    tags = ddb.tag(shot)
                    if tags['IsDisrupt']:
                        t1 = tags['CqTime']
                    else:
                        t1 = tags['RampDownTime']
                    new_dig_length = int(
                        (t1 * 1000 - self._start_time) * self._sample_rate)
                    data = data_reader.read_many(shot, self._tags)
                    digs = []
                    for tag, (dig, time) in data.items():
                        dig = dig[(self._start_time / 1000 <= time)
                                  & (time <= t1)]
                        # 归一化
                        if self._normalized:
                            dig = (dig - self._normalize_param[tag]['min']) / \
                                  (self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
                        # 重采样
                        digs.append(signal.resample(dig, new_dig_length))

                    digs = np.array(digs)

                    f = h5py.File(
                        os.path.join(self._directory, categories,
                                     '{}.hdf5'.format(shot)))
                    dataset = f.create_dataset('diagnosis', data=digs)
                    for key, value in tags.items():
                        dataset.attrs.create(key, value)
                    f.close()

                except Exception as e:
                    print(e)
                    traceback.print_exc()
예제 #8
0
 def TagNum(self, Taglist=None, Shotlist=None):
     Taglist  = list(Taglist)
     Shotlist = list(Shotlist)
     for i in range(len(Taglist)):
         Taglist[i] = Taglist[i][1:len(Taglist[i])]
     result = {}
     for i in Taglist:
         result.update({i: 0})
     reader = Reader(root_path=self.hdf5path)
     n = 1
     for i in Shotlist:
         print("第{}个".format(n))
         n += 1
         ShotTag = reader.tags(int(i))
         for k in range(len(ShotTag)):
             ShotTag[k] = ShotTag[k][1:len(ShotTag[k])]
         for j in Taglist:
             for tag in ShotTag:
                 if j == tag :
                     result[j] = result[j]+1
     pprint.pprint(result)
예제 #9
0
 def Vpfiltshot(self, Shotlist=None, Threshold=None, Detail=False):
     Shotlist = list(Shotlist)
     input = Shotlist
     if not Threshold:
         Threshold = 0.015
     reader = Reader(root_path=self.hdf5path)
     breakvp = []
     n = 1
     for shot in Shotlist:
         print("第{}个".format(n))
         n += 1
         data = reader.read_one(int(shot), r"\vp2")
         # 低通滤波
         ba = signal.butter(8, 0.01, "lowpass")
         fdata = signal.filtfilt(ba[0], ba[1], data[0])
         if max(fdata) < Threshold:
             breakvp.append(shot)
     output = breakvp
     if Detail:
         print("Number of input shots    : {}".format(len(input)))
         print("Number of output shots   : {}".format(len(output)))
     return output
예제 #10
0
파일: Plot.py 프로젝트: axk19970225/DDB
    def plot_much(self,
                  Taglist=None,
                  Shotlist=None,
                  Savepath=None,
                  ShowDownTime=False,
                  ShowIpFlat=False,
                  xline=None,
                  yline=None):
        if Savepath:
            root_path = Savepath
            if not os.path.exists(root_path):
                raise ValueError(
                    'No such saving path, you need to create one! ')
        else:
            root_path = os.getcwd() + os.sep + "plot"
            print(root_path)
            if not os.path.exists(root_path):
                os.makedirs(root_path)
        for tag in Taglist:
            tag_name = tag[1:]
            file_path = root_path + os.sep + tag_name
            if not os.path.exists(file_path):
                os.makedirs(file_path)

        reader = Reader(root_path=self.hdf5path)
        db = Query()
        for tag in Taglist:
            tag_name = tag[1:]
            file_path = root_path + os.sep + tag_name
            n = 1
            for shot in Shotlist:
                print("Shot:{}".format(shot) + " Tag:{}  ".format(tag_name) +
                      "No.{}".format(n))
                n += 1
                try:
                    shot_info = db.tag(int(shot))
                    data = reader.read_one(int(shot), tag)
                    plt.figure((str(shot) + tag_name))
                    plt.plot(data[1], data[0], 'g')
                    if ShowDownTime:
                        if shot_info["IsValidShot"]:
                            if shot_info["IsDisrupt"]:
                                plt.axvline(round(shot_info["CqTime"], 3),
                                            c='r')
                            else:
                                plt.axvline(round(shot_info["RampDownTime"],
                                                  3),
                                            c='r')
                    if ShowIpFlat:
                        if tag == r"\ip":
                            if shot_info["IsValidShot"]:
                                plt.axhline(round(shot_info["IpFlat"], 3),
                                            c='k')
                    if xline:
                        if not isNum(xline):
                            raise ValueError('xline needs to be number ')
                        plt.axvline(round(xline, 3))
                    if yline:
                        if not isNum(yline):
                            raise ValueError('yline needs to be number ')
                        plt.axhline(round(yline, 3))
                    path = file_path + os.sep + r"{}.png".format(shot)
                    plt.savefig(path)
                    plt.close()
                except Exception as err:
                    print("Shot:{}".format(shot) +
                          " Tag:{}  ".format(tag_name) + "No data")
                    plt.close()
                    pass
                      r"TrainNormal.npy")
TrainBreak = np.load(root_path + os.sep + r"TrainData" + os.sep +
                     r"TrainBreak.npy")
TrainNormal = list(TrainNormal)
TrainBreak = list(TrainBreak)
print(len(TrainNormal))
print(len(TrainBreak))
TrainShot = TrainNormal + TrainBreak

save_path = root_path + os.sep + r"ReduceSampling"
if not os.path.exists(save_path):
    os.makedirs(save_path)

n = 1
mistake = []
reader = Reader()
db = Query()
for shot in TrainShot:
    print("Shot:{}  ".format(shot) + "No.{}".format(n))
    n += 1
    try:
        shot_info = db.tag(int(shot))
        file = h5py.File(save_path + os.sep + r"{}.hdf5".format(shot))
        if not shot_info["IsDisrupt"]:
            DownTime = shot_info["RampDownTime"]
            for shottag in all_tags:
                dataset = reader.read_one(int(shot), shottag)
                data = dataset[0]
                time = dataset[1]
                data = data[time <= DownTime]
                time = time[time <= DownTime]