def __init__(self, etabs_model, json_file): super(Form, self).__init__() self.form = Gui.PySideUic.loadUi( str(civiltools_path / 'widgets' / 'earthquake_factor.ui')) self.json_file = json_file self.city = None self.etabs = etabs_model self.load_config() self.set_properties_from_json() self.final_building = self.current_building() self.structure_model = StructureModel(self.final_building) self.form.structure_properties_table.setModel(self.structure_model) self.create_connections() # self.load_settings() self.calculate() self.fill_dialog()
def run(config): import random hash = random.getrandbits(32) logger = logging.getLogger() logger.setLevel(logging.DEBUG) ah = logging.FileHandler(str(hash) + '.log') ah.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(message)s') ah.setFormatter(formatter) logger.addHandler(ah) num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data( config) print(embedding_matrix.shape) config.n_embed, config.d_embed = embedding_matrix.shape config.dim_hidden = config.dim_sem + config.dim_str print(config.__flags) logger.critical(str(config.__flags)) model = StructureModel(config) model.build() model.get_loss() # trainer = Trainer(config) num_batches_per_epoch = int(num_examples / config.batch_size) num_steps = config.epochs * num_batches_per_epoch saver = tf.train.Saver() with tf.Session() as sess: gvi = tf.global_variables_initializer() sess.run(gvi) sess.run(model.embeddings.assign(embedding_matrix.astype(np.float32))) loss = 0 for ct, batch in tqdm.tqdm(train_batches, total=num_steps): feed_dict = model.get_feed_dict(batch) outputs, _, _loss = sess.run( [model.final_output, model.opt, model.loss], feed_dict=feed_dict) loss += _loss if (ct % config.log_period == 0): acc_test = evaluate(sess, model, test_batches) acc_dev = evaluate(sess, model, dev_batches) print('Step: {} Loss: {}\n'.format(ct, loss)) print('Test ACC: {}\n'.format(acc_test)) print('Dev ACC: {}\n'.format(acc_dev)) logger.debug('Step: {} Loss: {}\n'.format(ct, loss)) logger.debug('Test ACC: {}\n'.format(acc_test)) logger.debug('Dev ACC: {}\n'.format(acc_dev)) logger.handlers[0].flush() loss = 0 saver.save(sess, 'my_test_model', global_step=1000)
def run(config): import random hash = random.getrandbits(32) # 32자리 비트를 랜덤 생성 # logging : DEBUG < INFO < *WARNING < *ERROR < *CRITICAL (앞의 두 레벨은 따로 설정해야 출력해준다.) # 1. 생성, 2. 레벨 설정, 3. (파일)핸들러 설정(내가 로깅한 정보가 (파일로) 출력되는 위치 설정하는 것), 4. 출력 포매팅 설정 # logger <- ah <- formatter logger = logging.getLogger() # 1. 자신만의 특정한 로거 만들기(루트로거 리턴, 기본레벨은 warning) logger.setLevel(logging.DEBUG) # 2. DEBUG까지 출력해줘라. ah = logging.FileHandler(str(hash) + '.log') # 3. 해당 디렉토리에 파일이 만들어진다. ah.setLevel(logging.DEBUG) # 3. 여기에서도 레벨을 정해준다. formatter = logging.Formatter( '%(asctime)s - %(message)s') # 정보 출력(시간, 메시지) ah.setFormatter(formatter) # 포매터를 일반 핸들러에도 붙이고 파일 핸들러에도 붙임. logger.addHandler(ah) # 로거에 만든 핸들러를 붙여줌. num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data( config) print(embedding_matrix.shape) config.n_embed, config.d_embed = embedding_matrix.shape # config 추가(n_embed, d_embed) config.dim_hidden = config.dim_sem + config.dim_str # config 추가(hidden = semantic_dimension + structure_dimension) print(config.__flags) # example : {'rnn_cell': <absl.flags._flag.Flag object at 0x7f16402168d0>, 'data_file': <absl.flags._flag.Flag object at 0x7f1640216e80>, 'batch_size': <absl.flags._flag.Flag object at 0x7f16142b09b0>, 'epochs': <absl.flags._flag.Flag object at 0x7f16142b0b00>, 'dim_str': <absl.flags._flag.Flag object at 0x7f16142b0ba8>, 'dim_sem': <absl.flags._flag.Flag object at 0x7f16142b0c88>, 'dim_output': <absl.flags._flag.Flag object at 0x7f16142b0cc0>, 'keep_prob': <absl.flags._flag.Flag object at 0x7f16142b0dd8>, 'opt': <absl.flags._flag.Flag object at 0x7f16142b0f28>, 'lr': <absl.flags._flag.Flag object at 0x7f16142b0f60>, 'norm': <absl.flags._flag.Flag object at 0x7f16142bb048>, 'gpu': <absl.flags._flag.Flag object at 0x7f16142bb0b8>, 'sent_attention': <absl.flags._flag.Flag object at 0x7f16142bb0f0>, 'doc_attention': <absl.flags._flag.Flag object at 0x7f16142bb1d0>, 'large_data': <absl.flags._flag.BooleanFlag object at 0x7f16142b0b38>, 'log_period': <absl.flags._flag.Flag object at 0x7f16142bb2e8>} logger.critical(str(config.__flags)) # 로그 출력 : logger.debug('debug'), logger.info('info'), logger.warn('warn'), logger.error('error'), logger.critical('critical') model = StructureModel(config) # config전달 받아서 model 객체를 생성한다. model.build() model.get_loss() # trainer = Trainer(config) num_batches_per_epoch = int( num_examples / config.batch_size ) # num_examples : len(train), example: 32/16 = 2-> 한 epoch에 2배피씩 처리 num_steps = config.epochs * num_batches_per_epoch # 30 * 2 = 60 -> 총 처리해야하는 스텝 with tf.Session() as sess: gvi = tf.global_variables_initializer() sess.run(gvi) sess.run(model.embeddings.assign(embedding_matrix.astype( np.float32))) # embedding_matrix : load_data에서 가지고 옴. loss = 0 for ct, batch in tqdm.tqdm(train_batches, total=num_steps): feed_dict = model.get_feed_dict(batch) outputs, _, _loss = sess.run( [model.final_output, model.opt, model.loss], feed_dict=feed_dict) loss += _loss if (ct % config.log_period == 0): # config,log_period = 5000 acc_test = evaluate(sess, model, test_batches) acc_dev = evaluate(sess, model, dev_batches) print('Step: {} Loss: {}\n'.format(ct, loss)) print('Test ACC: {}\n'.format(acc_test)) print('Dev ACC: {}\n'.format(acc_dev)) logger.debug('Step: {} Loss: {}\n'.format(ct, loss)) logger.debug('Test ACC: {}\n'.format(acc_test)) logger.debug('Dev ACC: {}\n'.format(acc_dev)) logger.handlers[0].flush() # ? loss = 0 # 5000단위로 loss본다.
def run(config): import random hash = random.getrandbits(32) logger = logging.getLogger() logger.setLevel(logging.DEBUG) ah = logging.FileHandler(str(hash) + '.log') ah.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(message)s') ah.setFormatter(formatter) logger.addHandler(ah) num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab, tag2id = load_data( config) id2tag = {v: k for k, v in tag2id.items()} print(embedding_matrix.shape) config.n_embed, config.d_embed = embedding_matrix.shape config.dim_hidden = config.dim_sem + config.dim_str config.dim_output = len(tag2id) print(config.__flags) logger.critical(str(config.__flags)) if config.arch == 'sa': model = StructureModel(config) elif config.arch == 'bs': model = BilstmSoftmax(config) elif config.arch == 'bs-crf': model = BilstmCRF(config) elif config.arch == 'sa-crf': model = StructureModelCRF(config) model.build() model.get_loss() # trainer = Trainer(config) num_batches_per_epoch = int(num_examples / config.batch_size) num_steps = config.epochs * num_batches_per_epoch with tf.Session() as sess: gvi = tf.global_variables_initializer() sess.run(gvi) sess.run(model.embeddings.assign(embedding_matrix.astype(np.float32))) loss = 0 for ct, batch in tqdm.tqdm(train_batches, total=num_steps): feed_dict = model.get_feed_dict(batch, id2tag) outputs, _, _loss = sess.run( [model.final_output, model.opt, model.loss], feed_dict=feed_dict) loss += _loss if (ct % config.log_period == 0): if config.arch in ['bs-crf', 'sa-crf']: acc_test = evaluate_crf(sess, model, test_batches, config.test_output, id2tag) acc_dev = evaluate_crf(sess, model, dev_batches, config.dev_output, id2tag) else: acc_test = evaluate(sess, model, test_batches, config.test_output, id2tag) acc_dev = evaluate(sess, model, dev_batches, config.dev_output, id2tag) print('Step: {} Loss: {}\n'.format(ct, loss)) print('Test ACC: {}\n'.format(acc_test)) print('Dev ACC: {}\n'.format(acc_dev)) logger.debug('Step: {} Loss: {}\n'.format(ct, loss)) logger.debug('Test ACC: {}\n'.format(acc_test)) logger.debug('Dev ACC: {}\n'.format(acc_dev)) logger.handlers[0].flush() loss = 0
class Form(QtWidgets.QWidget): def __init__(self, etabs_model, json_file): super(Form, self).__init__() self.form = Gui.PySideUic.loadUi( str(civiltools_path / 'widgets' / 'earthquake_factor.ui')) self.json_file = json_file self.city = None self.etabs = etabs_model self.load_config() self.set_properties_from_json() self.final_building = self.current_building() self.structure_model = StructureModel(self.final_building) self.form.structure_properties_table.setModel(self.structure_model) self.create_connections() # self.load_settings() self.calculate() self.fill_dialog() def create_connections(self): self.form.xTAnalaticalSpinBox.valueChanged.connect(self.calculate) self.form.yTAnalaticalSpinBox.valueChanged.connect(self.calculate) self.form.xTAnalaticalSpinBox.valueChanged.connect(self.set_bx) self.form.yTAnalaticalSpinBox.valueChanged.connect(self.set_by) self.form.apply_to_etabs.clicked.connect(self.apply_factors_to_etabs) self.form.export_to_word.clicked.connect(self.export_to_word) def set_bx(self): self.form.bx = self.final_building.Bx def set_by(self): self.form.by = self.final_building.By def create_widgets(self): self.load_config() # # # curve widget # self.curveBWidget = pl() # self.p = self.curveBWidget.p # self.curveBWidget.setMinimumSize(450, 300) # draw_layout = QVBoxLayout() # draw_layout.addWidget(self.curveBWidget) # self.draw_frame.setLayout(draw_layout) def accept(self): Gui.Control.closeDialog() def load_settings(self): qsettings = QSettings("civiltools", "cfactor") self.restoreGeometry(qsettings.value("geometry", self.saveGeometry())) self.form.hsplitter1.restoreState( qsettings.value("hsplitter1", self.form.hsplitter1.saveState())) def load_config(self): if self.json_file.exists(): tx, ty = config.get_analytical_periods(self.json_file) self.form.xTAnalaticalSpinBox.setValue(tx) self.form.yTAnalaticalSpinBox.setValue(ty) def save_config(self): tx = self.form.xTAnalaticalSpinBox.value() ty = self.form.yTAnalaticalSpinBox.value() config.save_analytical_periods(self.json_file, tx, ty) def getTAnalatical(self): xTan = self.form.xTAnalaticalSpinBox.value() yTan = self.form.yTAnalaticalSpinBox.value() return xTan, yTan def setSoilProperties(self, build=None): if not build: build = self.current_building() xrf = build.soil_reflection_prop_x yrf = build.soil_reflection_prop_y soilProp = [build.soilType, xrf.T0, xrf.Ts, xrf.S, xrf.S0] xSoilProp = [xrf.B1, xrf.N, build.Bx] ySoilProp = [yrf.B1, yrf.N, build.By] for row, item in enumerate(soilProp): if row == 0: item = QTableWidgetItem("%s " % item) else: item = QTableWidgetItem("%.2f " % item) item.setTextAlignment(Qt.AlignCenter) self.form.soilPropertiesTable.setItem(row, 0, item) for row, item in enumerate(xSoilProp): item = QTableWidgetItem("%.2f " % item) item.setTextAlignment(Qt.AlignCenter) self.form.soilPropertiesTable.setItem(row + len(soilProp), 0, item) for row, item in enumerate(ySoilProp): item = QTableWidgetItem("%.2f " % item) item.setTextAlignment(Qt.AlignCenter) self.form.soilPropertiesTable.setItem(row + len(soilProp), 1, item) def set_properties_from_json(self): if self.json_file is None: etabs_filename = self.etabs.get_filename() self.json_file = etabs_filename.with_suffix('.json') d = config.load(self.json_file) self.risk_level = d['risk_level'] self.height_x = d['height_x'] self.importance_factor = float(d['importance_factor']) self.soil = d['soil_type'] self.city = d['city'] self.noStory = d['no_of_story_x'] self.xSystemType = d['x_system_name'] self.xLateralType = d['x_lateral_name'] self.ySystemType = d['y_system_name'] self.yLateralType = d['y_lateral_name'] self.is_infill = d['infill'] self.xSystem = StructureSystem(self.xSystemType, self.xLateralType, "X") self.ySystem = StructureSystem(self.ySystemType, self.yLateralType, "Y") def fill_dialog(self): if self.city is not None: self.form.risk_level.setText(self.risk_level) self.form.importance_factor.setText(str(self.importance_factor)) self.form.rx.setText(str(self.final_building.x_system.Ru)) self.form.ry.setText(str(self.final_building.y_system.Ru)) self.form.bx.setText(str(self.final_building.Bx)) self.form.by.setText(str(self.final_building.By)) def current_building(self): if self.city is None: return None xTan, yTan = self.getTAnalatical() build = Building( self.risk_level, self.importance_factor, self.soil, self.noStory, self.height_x, self.is_infill, self.xSystem, self.ySystem, self.city, xTan, yTan, ) return build def calculate(self): self.dirty = False self.final_building = self.current_building() if not self.final_building: return # self.setSoilProperties(self.final_building) self.structure_model.beginResetModel() self.structure_model.build = self.final_building self.structure_model.endResetModel() results = self.final_building.results if results[0] is False: title, err, direction = results[1:] QMessageBox.critical(self, title % direction, str(err)) def apply_factors_to_etabs(self): d = config.load(self.json_file) bot_story = d.get("bot_x_combo", '') top_story = d.get("top_x_combo", '') ret = self.etabs.apply_cfactor_to_edb( self.final_building, bot_story, top_story, ) if ret == 1: msg = "Data can not be written to your Etabs file,\n If you want to correct this problem, try Run analysis." title = "Remove Error?" QMessageBox.information(None, title, msg) return msg = "Successfully written to Etabs." self.save_config() QMessageBox.information(None, "done", msg) def exportBCurveToImage(self): export_graph = export.ExportGraph(self, self.lastDirectory, self.p) export_graph.to_image() def exportBCurveToCsv(self): export_graph = export.ExportGraph(self, self.lastDirectory, self.p) export_graph.to_csv() def export_to_word(self): filters = "docx(*.docx)" directory = str(self.json_file.parent) filename, _ = QFileDialog.getSaveFileName(None, 'Export To Word', directory, filters) if filename == '': return if not filename.endswith(".docx"): filename += ".docx" from exporter import export_to_word as word word.export(self.final_building, filename)
def run(config): logger = logging.getLogger() logger.setLevel(logging.DEBUG) time_log = str(time.time()) ah = logging.FileHandler(time_log + '.log') ah.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(message)s') ah.setFormatter(formatter) logger.addHandler(ah) tf.set_random_seed(config.seed) initializer = utils.Initializer(config.init_seed) xavier_init = initializer.xavier_init() if config.model_dir: print(config) print("Tensorflow version: ", tf.__version__) print("Git version: ", get_git_revision_hash()) logger.critical(str(config)) logger.critical(get_git_revision_hash()) evaluate_pretrained_model(config, logger) else: logger.debug("Going to load data") num_examples, train_batches, dev_batches, test_batches, embedding_matrix, vocab = load_data( config) logger.debug("Finished loading data.") # save vocab to file utils.save_dict(vocab, time_log + '.dict') print("Embedding matrix size: ", embedding_matrix.shape) config.n_embed, config.d_embed = embedding_matrix.shape config.dim_hidden = config.dim_sem + config.dim_str print(config) logger.critical(str(config)) print("Tensorflow version: ", tf.__version__) print("Git version: ", get_git_revision_hash()) logger.critical(get_git_revision_hash()) model = StructureModel(config, xavier_init) model.build() model.get_loss() num_batches_per_epoch = int(num_examples / config.batch_size) num_steps = config.epochs * num_batches_per_epoch best_acc_dev = 0.0 #with tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)) as sess: with tf.Session() as sess: gvi = tf.global_variables_initializer() sess.run(gvi) sess.run( model.embeddings.assign(embedding_matrix.astype(np.float64))) if config.debug: sess = tf_debug.LocalCLIDebugWrapperSession(sess) loss = 0 for ct, batch in tqdm.tqdm(train_batches, total=num_steps): feed_dict = model.get_feed_dict(batch) outputs, _, _loss = sess.run( [model.final_output, model.opt, model.loss], feed_dict=feed_dict) loss += _loss if (ct % config.log_period == 0): acc_test = evaluate(sess, model, test_batches, logger) acc_dev = evaluate(sess, model, dev_batches, logger) print('\nStep: {} Loss: {}'.format(ct, loss)) print('Test ACC: {}'.format(acc_test)) print('Dev ACC: %s (%s)', acc_dev, best_acc_dev) logger.debug('\nStep: {} Loss: {}'.format(ct, loss)) logger.debug('Test ACC: {}'.format(acc_test)) logger.debug('Dev ACC: %s (%s)', acc_dev, best_acc_dev) logger.handlers[0].flush() loss = 0 if acc_dev > best_acc_dev: best_acc_dev = acc_dev save_model(sess, ct, model, logger, config)