def TCShotlist(self, Taglist=None, Shotlist=None, Detail=False):
Shotlist = list(Shotlist)
input = Shotlist.copy()
reader = Reader(root_path=self.hdf5path)
n = 1
IncompleteNumber = []
for i in Shotlist:
print("第{}个".format(n))
n += 1
ShotTag = reader.tags(int(i))
for j in Taglist:
if j in ShotTag:
continue
else:
IncompleteNumber.append(i)
break
for i in IncompleteNumber:
for j in Shotlist:
if int(i) == int(j):
Shotlist.remove(j)
outlist = Shotlist
if Detail:
print("Number of input shots : {}".format(len(input)))
print("Number of excluded shots : {}".format(len(IncompleteNumber)))
print("Number of output shots : {}".format(len(outlist)))
return outlist
def __init__(self, root=None):
"""connect to mdsplus server and init data importer"""
if root:
self.exporter = Exporter(root)
self.reader = Reader(root)
else:
self.exporter = Exporter()
self.reader = Reader()
# log
log_dir = os.path.abspath('') + os.sep + 'log'
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.logger = logging.getLogger(__name__)
self.logger.setLevel(level=logging.INFO)
handler = logging.FileHandler(
log_dir + os.sep + 'JTEXTDataExporter_log_{}.txt'.format(
time.strftime("%Y_%m_%d %H:%M:%S", time.localtime(
time.time()))))
handler.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
self.logger.addHandler(handler)
self.logger.addHandler(console)
def run(self):
reader = Reader(self.hdf5_path)
for plugin in self.plugins.values():
signals = plugin.requested_signal()
for shot in self.shots:
self.logger.info('Run module {} @shot {}.\n'.format(
plugin.__class__.__name__, shot))
try:
data = reader.read_many(shot=shot, tags=signals)
result = plugin.generate(data)
except Exception as e:
self.logger.info('An error occurs in module {} @shot {}.\n{}\n{}'.format(
plugin.__class__.__name__, shot, e, traceback.format_exc()))
result = {}
if self.output['type'] == "mongodb":
from pymongo import MongoClient
client = MongoClient(self.output['host'], int(self.output['port']))
db = client[self.output['database']]
if self.config.has_option('output', 'username'):
db.authenticate(self.config.get('output', 'username'), self.config.get('output', 'password'))
col = db[self.output['collection']]
result['shot'] = shot
col.update_one(
{"shot": shot},
{"$set": result},
upsert=True
)
elif self.output['type'] == "stdio":
print(result)
else:
raise RuntimeError('配置文件output type选项不正确,可选为:mongodb,stdio')
def run(self):
data_reader = Reader()
ddb = Query()
for shot in self._shots:
# if shot < 1065500 or shot > 1065599:
# continue
print(shot)
try:
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(0.05 <= time) & (time <= t1)]
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
index = 0
path = os.path.join(self._npy_path, '{}'.format(shot))
if not os.path.exists(path):
os.makedirs(path)
while index + self._frame_size <= new_dig_length:
frame = digs[:, index:index + self._frame_size]
y_frame = y_[index:index + self._frame_size]
np.save(
os.path.join(
path, 'x_{}.npy'.format(int(index / self._step))),
frame)
np.save(
os.path.join(
path, 'y_{}.npy'.format(int(index / self._step))),
y_frame)
index += self._step
if index + self._frame_size - new_dig_length < self._frame_size / 2:
frame = digs[:, new_dig_length -
self._frame_size:new_dig_length]
y_frame = y_[new_dig_length -
self._frame_size:new_dig_length]
np.save(
os.path.join(
path, 'x_{}.npy'.format(int(index / self._step))),
frame)
np.save(
os.path.join(
path, 'y_{}.npy'.format(int(index / self._step))),
y_frame)
except Exception as e:
print(e)
traceback.print_exc()
def save_full_npy(self, shots):
data_reader = Reader()
ddb = Query()
path = os.path.join(self._npy_path, 'full')
if not os.path.exists(path):
os.makedirs(path)
print('####Start generate val DataSet####')
for shot in shots:
try:
print(shot)
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(0.05 <= time) & (time <= t1)]
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
index = 0
x = list()
labels = list()
while index + self._frame_size <= new_dig_length:
frame = digs[:, index:index + self._frame_size]
y_frame = y_[index:index + self._frame_size]
# index += self.frame_size
x.append(frame)
labels.append(y_frame[-1])
index += self._step
x = np.array(x)
labels = np.array(labels)
np.save(os.path.join(path, 'x_{}.npy'.format(shot)), x)
np.save(os.path.join(path, 'y_{}.npy'.format(shot)), labels)
except Exception as e:
print(e)
traceback.print_exc()
def plot_one(self,
Tag=None,
Shot=None,
Savepath=None,
xline=None,
yline=None,
Showplot=False):
if Savepath:
root_path = Savepath
if not os.path.exists(root_path):
raise ValueError(
'No such saving path, you need to create one! ')
else:
root_path = os.getcwd() + os.sep + "plot"
print(root_path)
if not os.path.exists(root_path):
os.makedirs(root_path)
reader = Reader(root_path=self.hdf5path)
try:
data = reader.read_one(int(Shot), Tag)
tag_name = Tag[1:]
plt.figure((str(Shot) + tag_name))
plt.plot(data[1], data[0], 'g')
if xline:
if not isNum(xline):
raise ValueError('xline needs to be number ')
plt.axvline(round(xline, 3))
if yline:
if not isNum(yline):
raise ValueError('yline needs to be number ')
plt.axhline(round(yline, 3))
name = str(Shot) + r" " + tag_name
path = root_path + os.sep + r"{}.png".format(name)
plt.savefig(path)
if Showplot:
plt.show()
plt.close()
except Exception as err:
print("Shot:{}".format(shot) + " Tag:{} ".format(tag_name) +
"No data")
plt.close()
pass
def generate(self):
data_reader = Reader()
ddb = Query()
for categories, shots in self._shots.items():
if not os.path.exists(os.path.join(self._directory, categories)):
os.makedirs(os.path.join(self._directory, categories))
for shot in shots:
print(shot)
try:
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int(
(t1 * 1000 - self._start_time) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(self._start_time / 1000 <= time)
& (time <= t1)]
# 归一化
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
# 重采样
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
f = h5py.File(
os.path.join(self._directory, categories,
'{}.hdf5'.format(shot)))
dataset = f.create_dataset('diagnosis', data=digs)
for key, value in tags.items():
dataset.attrs.create(key, value)
f.close()
except Exception as e:
print(e)
traceback.print_exc()
def TagNum(self, Taglist=None, Shotlist=None):
Taglist = list(Taglist)
Shotlist = list(Shotlist)
for i in range(len(Taglist)):
Taglist[i] = Taglist[i][1:len(Taglist[i])]
result = {}
for i in Taglist:
result.update({i: 0})
reader = Reader(root_path=self.hdf5path)
n = 1
for i in Shotlist:
print("第{}个".format(n))
n += 1
ShotTag = reader.tags(int(i))
for k in range(len(ShotTag)):
ShotTag[k] = ShotTag[k][1:len(ShotTag[k])]
for j in Taglist:
for tag in ShotTag:
if j == tag :
result[j] = result[j]+1
pprint.pprint(result)
def Vpfiltshot(self, Shotlist=None, Threshold=None, Detail=False):
Shotlist = list(Shotlist)
input = Shotlist
if not Threshold:
Threshold = 0.015
reader = Reader(root_path=self.hdf5path)
breakvp = []
n = 1
for shot in Shotlist:
print("第{}个".format(n))
n += 1
data = reader.read_one(int(shot), r"\vp2")
# 低通滤波
ba = signal.butter(8, 0.01, "lowpass")
fdata = signal.filtfilt(ba[0], ba[1], data[0])
if max(fdata) < Threshold:
breakvp.append(shot)
output = breakvp
if Detail:
print("Number of input shots : {}".format(len(input)))
print("Number of output shots : {}".format(len(output)))
return output
def plot_much(self,
Taglist=None,
Shotlist=None,
Savepath=None,
ShowDownTime=False,
ShowIpFlat=False,
xline=None,
yline=None):
if Savepath:
root_path = Savepath
if not os.path.exists(root_path):
raise ValueError(
'No such saving path, you need to create one! ')
else:
root_path = os.getcwd() + os.sep + "plot"
print(root_path)
if not os.path.exists(root_path):
os.makedirs(root_path)
for tag in Taglist:
tag_name = tag[1:]
file_path = root_path + os.sep + tag_name
if not os.path.exists(file_path):
os.makedirs(file_path)
reader = Reader(root_path=self.hdf5path)
db = Query()
for tag in Taglist:
tag_name = tag[1:]
file_path = root_path + os.sep + tag_name
n = 1
for shot in Shotlist:
print("Shot:{}".format(shot) + " Tag:{} ".format(tag_name) +
"No.{}".format(n))
n += 1
try:
shot_info = db.tag(int(shot))
data = reader.read_one(int(shot), tag)
plt.figure((str(shot) + tag_name))
plt.plot(data[1], data[0], 'g')
if ShowDownTime:
if shot_info["IsValidShot"]:
if shot_info["IsDisrupt"]:
plt.axvline(round(shot_info["CqTime"], 3),
c='r')
else:
plt.axvline(round(shot_info["RampDownTime"],
3),
c='r')
if ShowIpFlat:
if tag == r"\ip":
if shot_info["IsValidShot"]:
plt.axhline(round(shot_info["IpFlat"], 3),
c='k')
if xline:
if not isNum(xline):
raise ValueError('xline needs to be number ')
plt.axvline(round(xline, 3))
if yline:
if not isNum(yline):
raise ValueError('yline needs to be number ')
plt.axhline(round(yline, 3))
path = file_path + os.sep + r"{}.png".format(shot)
plt.savefig(path)
plt.close()
except Exception as err:
print("Shot:{}".format(shot) +
" Tag:{} ".format(tag_name) + "No data")
plt.close()
pass
class JTEXTDataExporter:
def __init__(self, root=None):
"""connect to mdsplus server and init data importer"""
if root:
self.exporter = Exporter(root)
self.reader = Reader(root)
else:
self.exporter = Exporter()
self.reader = Reader()
# log
log_dir = os.path.abspath('') + os.sep + 'log'
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.logger = logging.getLogger(__name__)
self.logger.setLevel(level=logging.INFO)
handler = logging.FileHandler(
log_dir + os.sep + 'JTEXTDataExporter_log_{}.txt'.format(
time.strftime("%Y_%m_%d %H:%M:%S", time.localtime(
time.time()))))
handler.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
self.logger.addHandler(handler)
self.logger.addHandler(console)
def download(self, shots=None, tags=None, sample_rate=None):
"""Download J-TEXT data
@:param shots: shot list
@:param tags: tag list
@:param sr: sample rate for resample(kHz), if sr<sr0: do resample, else: do nothing
:return: null
"""
if shots is None or len(shots) == 0:
raise ValueError('Param Error, ' 'shot list can\'t be Empty!')
if tags is None or len(tags) == 0:
raise ValueError('Param Error, ' 'tag list can\'t be Empty!')
self.logger.info('Start download data')
for shot in shots:
try:
c = connection.Connection('211.67.27.245')
c.openTree('jtext', shot=shot)
for tag in tags:
try:
if tag in self.reader.tags(shot):
self.logger.info(
'shot:{}, tag:{}, already exits.'.format(
shot, tag))
continue
data = np.array(c.get(tag))
time = np.array(
c.get(r'DIM_OF(BUILD_PATH({}))'.format(tag)))
if data.shape[0] == 0 or time.shape[0] == 0:
self.logger.info(
'shot:{}, tag:{}, Shape of signal or dim_of(signal) is 0'
)
elif abs(data.shape[0] - time.shape[0]) > 10:
self.logger.info(
'shot:{}, tag:{}, Shape of signal and dim_of(signal) not equal'
)
else:
if 0 < data.shape[0] - time.shape[0] < 10:
data = data[:time.shape[0]]
if 0 < time.shape[0] - data.shape[0] < 10:
time = time[:data.shape[0]]
sr0 = int(len(data) /
(time[-1] - time[0]) / 10) * 10
if sample_rate and sample_rate < sr0 / 100:
num = sample_rate * (int(
round(time[-1] * 1000 - time[0] * 1000)))
data = signal.resample(data, num)
duration = (time[-1] - time[0]) / num
time = []
for i in range(num):
time.append(i * duration)
result = self.exporter.save(shot, tag, data, time)
if result == 0:
self.logger.info(
'shot:{}, tag:{}, Finish.'.format(
shot, tag))
elif result == -1:
self.logger.info(
'shot:{}, tag:{}, already exits.'.format(
shot, tag))
except Exception as e:
self.logger.info(
'shot:{}, tag:{}, error occurs:{}.'.format(
shot, tag, e))
c.closeTree('jtext', shot=shot)
c.disconnect()
except Exception as err:
self.logger.info('shot {} is Empty, reason: {}\n{}'.format(
shot, err, traceback.format_exc()))
if 'SS-W-SUCCESS' in '{}'.format(err):
self.logger.info('stop at shot {}\n{}'.format(
shot, traceback.format_exc()))
exit(-1)
r"TrainNormal.npy")
TrainBreak = np.load(root_path + os.sep + r"TrainData" + os.sep +
r"TrainBreak.npy")
TrainNormal = list(TrainNormal)
TrainBreak = list(TrainBreak)
print(len(TrainNormal))
print(len(TrainBreak))
TrainShot = TrainNormal + TrainBreak
save_path = root_path + os.sep + r"ReduceSampling"
if not os.path.exists(save_path):
os.makedirs(save_path)
n = 1
mistake = []
reader = Reader()
db = Query()
for shot in TrainShot:
print("Shot:{} ".format(shot) + "No.{}".format(n))
n += 1
try:
shot_info = db.tag(int(shot))
file = h5py.File(save_path + os.sep + r"{}.hdf5".format(shot))
if not shot_info["IsDisrupt"]:
DownTime = shot_info["RampDownTime"]
for shottag in all_tags:
dataset = reader.read_one(int(shot), shottag)
data = dataset[0]
time = dataset[1]
data = data[time <= DownTime]
time = time[time <= DownTime]