def __init__(self, etabs_model, json_file):
super(Form, self).__init__()
self.form = Gui.PySideUic.loadUi(
str(civiltools_path / 'widgets' / 'earthquake_factor.ui'))
self.json_file = json_file
self.city = None
self.etabs = etabs_model
self.load_config()
self.set_properties_from_json()
self.final_building = self.current_building()
self.structure_model = StructureModel(self.final_building)
self.form.structure_properties_table.setModel(self.structure_model)
self.create_connections()
# self.load_settings()
self.calculate()
self.fill_dialog()
def run(config):
import random
hash = random.getrandbits(32)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
ah = logging.FileHandler(str(hash) + '.log')
ah.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
ah.setFormatter(formatter)
logger.addHandler(ah)
num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data(
config)
print(embedding_matrix.shape)
config.n_embed, config.d_embed = embedding_matrix.shape
config.dim_hidden = config.dim_sem + config.dim_str
print(config.__flags)
logger.critical(str(config.__flags))
model = StructureModel(config)
model.build()
model.get_loss()
# trainer = Trainer(config)
num_batches_per_epoch = int(num_examples / config.batch_size)
num_steps = config.epochs * num_batches_per_epoch
saver = tf.train.Saver()
with tf.Session() as sess:
gvi = tf.global_variables_initializer()
sess.run(gvi)
sess.run(model.embeddings.assign(embedding_matrix.astype(np.float32)))
loss = 0
for ct, batch in tqdm.tqdm(train_batches, total=num_steps):
feed_dict = model.get_feed_dict(batch)
outputs, _, _loss = sess.run(
[model.final_output, model.opt, model.loss],
feed_dict=feed_dict)
loss += _loss
if (ct % config.log_period == 0):
acc_test = evaluate(sess, model, test_batches)
acc_dev = evaluate(sess, model, dev_batches)
print('Step: {} Loss: {}\n'.format(ct, loss))
print('Test ACC: {}\n'.format(acc_test))
print('Dev ACC: {}\n'.format(acc_dev))
logger.debug('Step: {} Loss: {}\n'.format(ct, loss))
logger.debug('Test ACC: {}\n'.format(acc_test))
logger.debug('Dev ACC: {}\n'.format(acc_dev))
logger.handlers[0].flush()
loss = 0
saver.save(sess, 'my_test_model', global_step=1000)
def run(config):
import random
hash = random.getrandbits(32) # 32자리 비트를 랜덤 생성
# logging : DEBUG < INFO < *WARNING < *ERROR < *CRITICAL (앞의 두 레벨은 따로 설정해야 출력해준다.)
# 1. 생성, 2. 레벨 설정, 3. (파일)핸들러 설정(내가 로깅한 정보가 (파일로) 출력되는 위치 설정하는 것), 4. 출력 포매팅 설정
# logger <- ah <- formatter
logger = logging.getLogger() # 1. 자신만의 특정한 로거 만들기(루트로거 리턴, 기본레벨은 warning)
logger.setLevel(logging.DEBUG) # 2. DEBUG까지 출력해줘라.
ah = logging.FileHandler(str(hash) + '.log') # 3. 해당 디렉토리에 파일이 만들어진다.
ah.setLevel(logging.DEBUG) # 3. 여기에서도 레벨을 정해준다.
formatter = logging.Formatter(
'%(asctime)s - %(message)s') # 정보 출력(시간, 메시지)
ah.setFormatter(formatter) # 포매터를 일반 핸들러에도 붙이고 파일 핸들러에도 붙임.
logger.addHandler(ah) # 로거에 만든 핸들러를 붙여줌.
num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data(
config)
print(embedding_matrix.shape)
config.n_embed, config.d_embed = embedding_matrix.shape # config 추가(n_embed, d_embed)
config.dim_hidden = config.dim_sem + config.dim_str # config 추가(hidden = semantic_dimension + structure_dimension)
print(config.__flags)
# example : {'rnn_cell': <absl.flags._flag.Flag object at 0x7f16402168d0>, 'data_file': <absl.flags._flag.Flag object at 0x7f1640216e80>, 'batch_size': <absl.flags._flag.Flag object at 0x7f16142b09b0>, 'epochs': <absl.flags._flag.Flag object at 0x7f16142b0b00>, 'dim_str': <absl.flags._flag.Flag object at 0x7f16142b0ba8>, 'dim_sem': <absl.flags._flag.Flag object at 0x7f16142b0c88>, 'dim_output': <absl.flags._flag.Flag object at 0x7f16142b0cc0>, 'keep_prob': <absl.flags._flag.Flag object at 0x7f16142b0dd8>, 'opt': <absl.flags._flag.Flag object at 0x7f16142b0f28>, 'lr': <absl.flags._flag.Flag object at 0x7f16142b0f60>, 'norm': <absl.flags._flag.Flag object at 0x7f16142bb048>, 'gpu': <absl.flags._flag.Flag object at 0x7f16142bb0b8>, 'sent_attention': <absl.flags._flag.Flag object at 0x7f16142bb0f0>, 'doc_attention': <absl.flags._flag.Flag object at 0x7f16142bb1d0>, 'large_data': <absl.flags._flag.BooleanFlag object at 0x7f16142b0b38>, 'log_period': <absl.flags._flag.Flag object at 0x7f16142bb2e8>}
logger.critical(str(config.__flags))
# 로그 출력 : logger.debug('debug'), logger.info('info'), logger.warn('warn'), logger.error('error'), logger.critical('critical')
model = StructureModel(config) # config전달 받아서 model 객체를 생성한다.
model.build()
model.get_loss()
# trainer = Trainer(config)
num_batches_per_epoch = int(
num_examples / config.batch_size
) # num_examples : len(train), example: 32/16 = 2-> 한 epoch에 2배피씩 처리
num_steps = config.epochs * num_batches_per_epoch # 30 * 2 = 60 -> 총 처리해야하는 스텝
with tf.Session() as sess:
gvi = tf.global_variables_initializer()
sess.run(gvi)
sess.run(model.embeddings.assign(embedding_matrix.astype(
np.float32))) # embedding_matrix : load_data에서 가지고 옴.
loss = 0
for ct, batch in tqdm.tqdm(train_batches, total=num_steps):
feed_dict = model.get_feed_dict(batch)
outputs, _, _loss = sess.run(
[model.final_output, model.opt, model.loss],
feed_dict=feed_dict)
loss += _loss
if (ct % config.log_period == 0): # config,log_period = 5000
acc_test = evaluate(sess, model, test_batches)
acc_dev = evaluate(sess, model, dev_batches)
print('Step: {} Loss: {}\n'.format(ct, loss))
print('Test ACC: {}\n'.format(acc_test))
print('Dev ACC: {}\n'.format(acc_dev))
logger.debug('Step: {} Loss: {}\n'.format(ct, loss))
logger.debug('Test ACC: {}\n'.format(acc_test))
logger.debug('Dev ACC: {}\n'.format(acc_dev))
logger.handlers[0].flush() # ?
loss = 0 # 5000단위로 loss본다.
def run(config):
import random
hash = random.getrandbits(32)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
ah = logging.FileHandler(str(hash) + '.log')
ah.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
ah.setFormatter(formatter)
logger.addHandler(ah)
num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab, tag2id = load_data(
config)
id2tag = {v: k for k, v in tag2id.items()}
print(embedding_matrix.shape)
config.n_embed, config.d_embed = embedding_matrix.shape
config.dim_hidden = config.dim_sem + config.dim_str
config.dim_output = len(tag2id)
print(config.__flags)
logger.critical(str(config.__flags))
if config.arch == 'sa': model = StructureModel(config)
elif config.arch == 'bs': model = BilstmSoftmax(config)
elif config.arch == 'bs-crf': model = BilstmCRF(config)
elif config.arch == 'sa-crf': model = StructureModelCRF(config)
model.build()
model.get_loss()
# trainer = Trainer(config)
num_batches_per_epoch = int(num_examples / config.batch_size)
num_steps = config.epochs * num_batches_per_epoch
with tf.Session() as sess:
gvi = tf.global_variables_initializer()
sess.run(gvi)
sess.run(model.embeddings.assign(embedding_matrix.astype(np.float32)))
loss = 0
for ct, batch in tqdm.tqdm(train_batches, total=num_steps):
feed_dict = model.get_feed_dict(batch, id2tag)
outputs, _, _loss = sess.run(
[model.final_output, model.opt, model.loss],
feed_dict=feed_dict)
loss += _loss
if (ct % config.log_period == 0):
if config.arch in ['bs-crf', 'sa-crf']:
acc_test = evaluate_crf(sess, model, test_batches,
config.test_output, id2tag)
acc_dev = evaluate_crf(sess, model, dev_batches,
config.dev_output, id2tag)
else:
acc_test = evaluate(sess, model, test_batches,
config.test_output, id2tag)
acc_dev = evaluate(sess, model, dev_batches,
config.dev_output, id2tag)
print('Step: {} Loss: {}\n'.format(ct, loss))
print('Test ACC: {}\n'.format(acc_test))
print('Dev ACC: {}\n'.format(acc_dev))
logger.debug('Step: {} Loss: {}\n'.format(ct, loss))
logger.debug('Test ACC: {}\n'.format(acc_test))
logger.debug('Dev ACC: {}\n'.format(acc_dev))
logger.handlers[0].flush()
loss = 0
class Form(QtWidgets.QWidget):
def __init__(self, etabs_model, json_file):
super(Form, self).__init__()
self.form = Gui.PySideUic.loadUi(
str(civiltools_path / 'widgets' / 'earthquake_factor.ui'))
self.json_file = json_file
self.city = None
self.etabs = etabs_model
self.load_config()
self.set_properties_from_json()
self.final_building = self.current_building()
self.structure_model = StructureModel(self.final_building)
self.form.structure_properties_table.setModel(self.structure_model)
self.create_connections()
# self.load_settings()
self.calculate()
self.fill_dialog()
def create_connections(self):
self.form.xTAnalaticalSpinBox.valueChanged.connect(self.calculate)
self.form.yTAnalaticalSpinBox.valueChanged.connect(self.calculate)
self.form.xTAnalaticalSpinBox.valueChanged.connect(self.set_bx)
self.form.yTAnalaticalSpinBox.valueChanged.connect(self.set_by)
self.form.apply_to_etabs.clicked.connect(self.apply_factors_to_etabs)
self.form.export_to_word.clicked.connect(self.export_to_word)
def set_bx(self):
self.form.bx = self.final_building.Bx
def set_by(self):
self.form.by = self.final_building.By
def create_widgets(self):
self.load_config()
#
# # curve widget
# self.curveBWidget = pl()
# self.p = self.curveBWidget.p
# self.curveBWidget.setMinimumSize(450, 300)
# draw_layout = QVBoxLayout()
# draw_layout.addWidget(self.curveBWidget)
# self.draw_frame.setLayout(draw_layout)
def accept(self):
Gui.Control.closeDialog()
def load_settings(self):
qsettings = QSettings("civiltools", "cfactor")
self.restoreGeometry(qsettings.value("geometry", self.saveGeometry()))
self.form.hsplitter1.restoreState(
qsettings.value("hsplitter1", self.form.hsplitter1.saveState()))
def load_config(self):
if self.json_file.exists():
tx, ty = config.get_analytical_periods(self.json_file)
self.form.xTAnalaticalSpinBox.setValue(tx)
self.form.yTAnalaticalSpinBox.setValue(ty)
def save_config(self):
tx = self.form.xTAnalaticalSpinBox.value()
ty = self.form.yTAnalaticalSpinBox.value()
config.save_analytical_periods(self.json_file, tx, ty)
def getTAnalatical(self):
xTan = self.form.xTAnalaticalSpinBox.value()
yTan = self.form.yTAnalaticalSpinBox.value()
return xTan, yTan
def setSoilProperties(self, build=None):
if not build:
build = self.current_building()
xrf = build.soil_reflection_prop_x
yrf = build.soil_reflection_prop_y
soilProp = [build.soilType, xrf.T0, xrf.Ts, xrf.S, xrf.S0]
xSoilProp = [xrf.B1, xrf.N, build.Bx]
ySoilProp = [yrf.B1, yrf.N, build.By]
for row, item in enumerate(soilProp):
if row == 0:
item = QTableWidgetItem("%s " % item)
else:
item = QTableWidgetItem("%.2f " % item)
item.setTextAlignment(Qt.AlignCenter)
self.form.soilPropertiesTable.setItem(row, 0, item)
for row, item in enumerate(xSoilProp):
item = QTableWidgetItem("%.2f " % item)
item.setTextAlignment(Qt.AlignCenter)
self.form.soilPropertiesTable.setItem(row + len(soilProp), 0, item)
for row, item in enumerate(ySoilProp):
item = QTableWidgetItem("%.2f " % item)
item.setTextAlignment(Qt.AlignCenter)
self.form.soilPropertiesTable.setItem(row + len(soilProp), 1, item)
def set_properties_from_json(self):
if self.json_file is None:
etabs_filename = self.etabs.get_filename()
self.json_file = etabs_filename.with_suffix('.json')
d = config.load(self.json_file)
self.risk_level = d['risk_level']
self.height_x = d['height_x']
self.importance_factor = float(d['importance_factor'])
self.soil = d['soil_type']
self.city = d['city']
self.noStory = d['no_of_story_x']
self.xSystemType = d['x_system_name']
self.xLateralType = d['x_lateral_name']
self.ySystemType = d['y_system_name']
self.yLateralType = d['y_lateral_name']
self.is_infill = d['infill']
self.xSystem = StructureSystem(self.xSystemType, self.xLateralType,
"X")
self.ySystem = StructureSystem(self.ySystemType, self.yLateralType,
"Y")
def fill_dialog(self):
if self.city is not None:
self.form.risk_level.setText(self.risk_level)
self.form.importance_factor.setText(str(self.importance_factor))
self.form.rx.setText(str(self.final_building.x_system.Ru))
self.form.ry.setText(str(self.final_building.y_system.Ru))
self.form.bx.setText(str(self.final_building.Bx))
self.form.by.setText(str(self.final_building.By))
def current_building(self):
if self.city is None:
return None
xTan, yTan = self.getTAnalatical()
build = Building(
self.risk_level,
self.importance_factor,
self.soil,
self.noStory,
self.height_x,
self.is_infill,
self.xSystem,
self.ySystem,
self.city,
xTan,
yTan,
)
return build
def calculate(self):
self.dirty = False
self.final_building = self.current_building()
if not self.final_building:
return
# self.setSoilProperties(self.final_building)
self.structure_model.beginResetModel()
self.structure_model.build = self.final_building
self.structure_model.endResetModel()
results = self.final_building.results
if results[0] is False:
title, err, direction = results[1:]
QMessageBox.critical(self, title % direction, str(err))
def apply_factors_to_etabs(self):
d = config.load(self.json_file)
bot_story = d.get("bot_x_combo", '')
top_story = d.get("top_x_combo", '')
ret = self.etabs.apply_cfactor_to_edb(
self.final_building,
bot_story,
top_story,
)
if ret == 1:
msg = "Data can not be written to your Etabs file,\n If you want to correct this problem, try Run analysis."
title = "Remove Error?"
QMessageBox.information(None, title, msg)
return
msg = "Successfully written to Etabs."
self.save_config()
QMessageBox.information(None, "done", msg)
def exportBCurveToImage(self):
export_graph = export.ExportGraph(self, self.lastDirectory, self.p)
export_graph.to_image()
def exportBCurveToCsv(self):
export_graph = export.ExportGraph(self, self.lastDirectory, self.p)
export_graph.to_csv()
def export_to_word(self):
filters = "docx(*.docx)"
directory = str(self.json_file.parent)
filename, _ = QFileDialog.getSaveFileName(None, 'Export To Word',
directory, filters)
if filename == '':
return
if not filename.endswith(".docx"):
filename += ".docx"
from exporter import export_to_word as word
word.export(self.final_building, filename)
def run(config):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
time_log = str(time.time())
ah = logging.FileHandler(time_log + '.log')
ah.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
ah.setFormatter(formatter)
logger.addHandler(ah)
tf.set_random_seed(config.seed)
initializer = utils.Initializer(config.init_seed)
xavier_init = initializer.xavier_init()
if config.model_dir:
print(config)
print("Tensorflow version: ", tf.__version__)
print("Git version: ", get_git_revision_hash())
logger.critical(str(config))
logger.critical(get_git_revision_hash())
evaluate_pretrained_model(config, logger)
else:
logger.debug("Going to load data")
num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data(
config)
logger.debug("Finished loading data.")
# save vocab to file
utils.save_dict(vocab, time_log + '.dict')
print("Embedding matrix size: ", embedding_matrix.shape)
config.n_embed, config.d_embed = embedding_matrix.shape
config.dim_hidden = config.dim_sem + config.dim_str
print(config)
logger.critical(str(config))
print("Tensorflow version: ", tf.__version__)
print("Git version: ", get_git_revision_hash())
logger.critical(get_git_revision_hash())
model = StructureModel(config, xavier_init)
model.build()
model.get_loss()
num_batches_per_epoch = int(num_examples / config.batch_size)
num_steps = config.epochs * num_batches_per_epoch
best_acc_dev = 0.0
#with tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)) as sess:
with tf.Session() as sess:
gvi = tf.global_variables_initializer()
sess.run(gvi)
sess.run(
model.embeddings.assign(embedding_matrix.astype(np.float64)))
if config.debug:
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
loss = 0
for ct, batch in tqdm.tqdm(train_batches, total=num_steps):
feed_dict = model.get_feed_dict(batch)
outputs, _, _loss = sess.run(
[model.final_output, model.opt, model.loss],
feed_dict=feed_dict)
loss += _loss
if (ct % config.log_period == 0):
acc_test = evaluate(sess, model, test_batches, logger)
acc_dev = evaluate(sess, model, dev_batches, logger)
print('\nStep: {} Loss: {}'.format(ct, loss))
print('Test ACC: {}'.format(acc_test))
print('Dev ACC: %s (%s)', acc_dev, best_acc_dev)
logger.debug('\nStep: {} Loss: {}'.format(ct, loss))
logger.debug('Test ACC: {}'.format(acc_test))
logger.debug('Dev ACC: %s (%s)', acc_dev, best_acc_dev)
logger.handlers[0].flush()
loss = 0
if acc_dev > best_acc_dev:
best_acc_dev = acc_dev
save_model(sess, ct, model, logger, config)