def run(self):
reader = Reader(self.hdf5_path)
for plugin in self.plugins.values():
signals = plugin.requested_signal()
for shot in self.shots:
self.logger.info('Run module {} @shot {}.\n'.format(
plugin.__class__.__name__, shot))
try:
data = reader.read_many(shot=shot, tags=signals)
result = plugin.generate(data)
except Exception as e:
self.logger.info('An error occurs in module {} @shot {}.\n{}\n{}'.format(
plugin.__class__.__name__, shot, e, traceback.format_exc()))
result = {}
if self.output['type'] == "mongodb":
from pymongo import MongoClient
client = MongoClient(self.output['host'], int(self.output['port']))
db = client[self.output['database']]
if self.config.has_option('output', 'username'):
db.authenticate(self.config.get('output', 'username'), self.config.get('output', 'password'))
col = db[self.output['collection']]
result['shot'] = shot
col.update_one(
{"shot": shot},
{"$set": result},
upsert=True
)
elif self.output['type'] == "stdio":
print(result)
else:
raise RuntimeError('配置文件output type选项不正确,可选为:mongodb,stdio')
def run(self):
data_reader = Reader()
ddb = Query()
for shot in self._shots:
# if shot < 1065500 or shot > 1065599:
# continue
print(shot)
try:
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(0.05 <= time) & (time <= t1)]
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
index = 0
path = os.path.join(self._npy_path, '{}'.format(shot))
if not os.path.exists(path):
os.makedirs(path)
while index + self._frame_size <= new_dig_length:
frame = digs[:, index:index + self._frame_size]
y_frame = y_[index:index + self._frame_size]
np.save(
os.path.join(
path, 'x_{}.npy'.format(int(index / self._step))),
frame)
np.save(
os.path.join(
path, 'y_{}.npy'.format(int(index / self._step))),
y_frame)
index += self._step
if index + self._frame_size - new_dig_length < self._frame_size / 2:
frame = digs[:, new_dig_length -
self._frame_size:new_dig_length]
y_frame = y_[new_dig_length -
self._frame_size:new_dig_length]
np.save(
os.path.join(
path, 'x_{}.npy'.format(int(index / self._step))),
frame)
np.save(
os.path.join(
path, 'y_{}.npy'.format(int(index / self._step))),
y_frame)
except Exception as e:
print(e)
traceback.print_exc()
def save_full_npy(self, shots):
data_reader = Reader()
ddb = Query()
path = os.path.join(self._npy_path, 'full')
if not os.path.exists(path):
os.makedirs(path)
print('####Start generate val DataSet####')
for shot in shots:
try:
print(shot)
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int((t1 * 1000 - 50) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(0.05 <= time) & (time <= t1)]
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
y_ = y(new_dig_length, self._sample_rate, tags['IsDisrupt'])
index = 0
x = list()
labels = list()
while index + self._frame_size <= new_dig_length:
frame = digs[:, index:index + self._frame_size]
y_frame = y_[index:index + self._frame_size]
# index += self.frame_size
x.append(frame)
labels.append(y_frame[-1])
index += self._step
x = np.array(x)
labels = np.array(labels)
np.save(os.path.join(path, 'x_{}.npy'.format(shot)), x)
np.save(os.path.join(path, 'y_{}.npy'.format(shot)), labels)
except Exception as e:
print(e)
traceback.print_exc()
def generate(self):
data_reader = Reader()
ddb = Query()
for categories, shots in self._shots.items():
if not os.path.exists(os.path.join(self._directory, categories)):
os.makedirs(os.path.join(self._directory, categories))
for shot in shots:
print(shot)
try:
tags = ddb.tag(shot)
if tags['IsDisrupt']:
t1 = tags['CqTime']
else:
t1 = tags['RampDownTime']
new_dig_length = int(
(t1 * 1000 - self._start_time) * self._sample_rate)
data = data_reader.read_many(shot, self._tags)
digs = []
for tag, (dig, time) in data.items():
dig = dig[(self._start_time / 1000 <= time)
& (time <= t1)]
# 归一化
if self._normalized:
dig = (dig - self._normalize_param[tag]['min']) / \
(self._normalize_param[tag]['max'] - self._normalize_param[tag]['min'])
# 重采样
digs.append(signal.resample(dig, new_dig_length))
digs = np.array(digs)
f = h5py.File(
os.path.join(self._directory, categories,
'{}.hdf5'.format(shot)))
dataset = f.create_dataset('diagnosis', data=digs)
for key, value in tags.items():
dataset.attrs.create(key, value)
f.close()
except Exception as e:
print(e)
traceback.print_exc()