def __init__(self,base_model_list=bagging_config.base_model_list): self.base_model_list = base_model_list.split("-") self.num_random=len(self.base_model_list) self.dataDir = general_config.data_dir + "/random" createRandomData(self.num_random) self.models = [] self.models_name=[] for i in range(self.num_random): base_model = self.base_model_list[i] assert base_model in ["1", "2", "3", "4","5"], "Invalid base model type!" if base_model == "1": model = TextCNN() elif base_model == "2": model = TextRNN() elif base_model == "3": model = CRNN() elif base_model=="4": model = RCNN() else: model=HAN() self.models.append(model) self.models_name.append(modelDict[base_model]) self.logDir = ensure_dir_exist(general_config.log_dir + "/bagging/" + "-".join(self.models_name)) self.saveDir = ensure_dir_exist(general_config.save_dir + "/bagging/" + "-".join(self.models_name)) self.logger=my_logger(self.logDir+"/log.txt")
def __init__(self): self.embeddings = load_embedding_matrix( wv_path=general_config.wv_path, int2vocabPath=general_config.global_static_i2v_path) self.model = GaussianNB() self.log_dir = ensure_dir_exist(general_config.log_dir + "/NB") self.save_dir = ensure_dir_exist(general_config.save_dir + "/NB") self.logger = my_logger(self.log_dir + "/log.txt")
def predict(self, testFile=None, vocab2intPath=None, load_path=general_config.load_path_test, is_save=True, resPath=None): if testFile is None or vocab2intPath is None: testFile = os.path.join(general_config.data_dir, "testing_data_new.txt") vocab2intPath = general_config.global_nonstatic_v2i_path test_generator = BucketedDataIteratorForDoc( loadPath=testFile, vocab2intPath=vocab2intPath) load_dir = load_path if os.path.isdir(load_path) else os.path.dirname( load_path) log_dir = load_dir.replace("checkpoints", "logs") logger = my_logger(log_dir + "/log_predict.txt") os.environ['CUDA_VISIBLE_DEVICES'] = str(0) config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.8 with tf.Session(config=config, graph=self.graph) as sess: logger.info("Loading model...") saver = tf.train.Saver() if os.path.isdir(load_path): ckpt = tf.train.get_checkpoint_state(load_path) saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split("-")[-1] else: saver.restore(sess, load_path) global_step = load_path.split("-")[-1] logger.info("Loading successfully, loading epoch is %s" % global_step) cur_loop = test_generator.loop batch_idx, batch_docs, _, batch_doc_lens, batch_sent_lens = test_generator.next( batch_size=1024, need_all=True) res = {} while (test_generator.loop == cur_loop): predicted = sess.run(self.predicted, feed_dict=self._feed_dict_test( batch_docs, batch_doc_len=batch_doc_lens, batch_sent_len=batch_sent_lens)) for (id, label) in zip(batch_idx, predicted): res[id] = int(label) batch_idx, batch_docs, _, batch_doc_lens, batch_sent_lens = test_generator.next( 1024, need_all=True) if is_save: if resPath is None: res_dir = ensure_dir_exist( load_dir.replace("checkpoints", "results")) resPath = os.path.join(res_dir, "predicted.csv-" + str(global_step)) res_save = [[key, value] for (key, value) in res.items()] # 用于存放测试识别结果 WriteToSubmission(fileName=resPath, res=res_save) return res
def evaluate(self, load_path=general_config.load_path_test, validFile=None, vocab2intPath=None): if validFile is None or vocab2intPath is None: validFile = general_config.training_file vocab2intPath = general_config.global_nonstatic_v2i_path train_generator = PaddedDataIterator(loadPath=validFile, vocab2intPath=vocab2intPath, sent_len_cut=self.min_len) load_dir = load_path if os.path.isdir(load_path) else os.path.dirname( load_path) log_dir = load_dir.replace("checkpoints", "logs") logger = my_logger(log_dir + "/log_evaluate.txt") os.environ['CUDA_VISIBLE_DEVICES'] = str(0) config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.8 with tf.Session(config=config, graph=self.graph) as sess: logger.info("Loading model...") saver = tf.train.Saver() if os.path.isdir(load_path): ckpt = tf.train.get_checkpoint_state(load_path) saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split("-")[-1] else: saver.restore(sess, load_path) global_step = load_path.split("-")[-1] logger.info("Loading successfully, loading epoch is %s" % global_step) logger.info("The total number of trainable variables: %s" % get_num_params()) cur_loop = train_generator.loop cur_count = 0 avg_loss_t, avg_accuracy_t = 0., 0. _, batch_seqs, batch_labels, batch_lens = train_generator.next( 1024, need_all=True) while (train_generator.loop == cur_loop): cur_count += 1 loss_t, acc_t = sess.run([self.loss_op, self.acc_op], feed_dict=self._feed_dict_valid( batch_seqs, batch_labels, batch_lens)) avg_loss_t += loss_t avg_accuracy_t += acc_t _, batch_seqs, batch_labels, batch_lens = train_generator.next( 1024, need_all=True) avg_loss_t /= cur_count avg_accuracy_t /= cur_count logger.info("Loss: %.4f, Accuracy: %.4f " % (avg_loss_t, avg_accuracy_t)) return avg_loss_t, avg_accuracy_t
def disconnect(dc,vm_name): logger=my_logger() if dc == 'two': vcent_user,vcent_pass,vcent_ip = settings.TVCENTER_USER, settings.TVCENTER_PASSWORD, settings.TVCENTER_IP elif dc == 'san': vcent_user,vcent_pass,vcent_ip = settings.SVCENTER_USER, settings.SVCENTER_PASSWORD, settings.SVCENTER_IP else: vcent_user,vcent_pass,vcent_ip = settings.FVCENTER_USER, settings.FVCENTER_PASSWORD, settings.FVCENTER_IP dis='govc device.disconnect -u "%s":"%s"@"%s" -k -dc="%s" -vm "%s" cdrom-3000' \ %(vcent_user,vcent_pass,vcent_ip,dc,vm_name) stdout, stderr = bash(dis) if stderr: logger.error('机房: %s VM_Name: %s cdrom断开连接失败 %s' % (dc, vm_name, stderr.decode('utf-8'))) else: logger.info('机房: %s VM_Name: %s cdrom已断开连接' % (dc, vm_name))
def __init__(self, base_model_list=stacking_config.base_model_list, num_cv=stacking_config.num_cv): self.base_model_list = base_model_list.split("-") self.num_models = len(self.base_model_list) self.num_cv = num_cv self.dataDir = general_config.data_dir + "/cv/" + str(self.num_cv) if not os.path.exists(self.dataDir): createCrossValidationData(self.num_cv) self.models = [] self.models_name = [] for n in range(self.num_models): base_model = self.base_model_list[n] assert base_model in ["1", "2", "3", "4", "5"], "Invalid base model type!" if base_model == "1": model = TextCNN() elif base_model == "2": model = TextRNN() elif base_model == "3": model = CRNN() elif base_model == "4": model = RCNN() else: model = HAN() self.models.append(model) self.models_name.append(modelDict[base_model]) self.logDir = ensure_dir_exist(general_config.log_dir + "/stacking/" + "-".join(self.models_name) + "/" + str(self.num_cv)) self.saveDir = ensure_dir_exist(general_config.save_dir + "/stacking/" + "-".join(self.models_name) + "/" + str(self.num_cv)) self.classifier = LogisticRegression() self.logger = my_logger(self.logDir + "/log.txt")
#! /usr/bin/env python3 from datetime import datetime import PyLora from led import led import os from utils import my_logger logger = my_logger(__name__, level="DEBUG") root_path = os.path.dirname(os.path.abspath(__file__)) PyLora.init() PyLora.set_frequency(433000000) PyLora.enable_crc() def csv_log_create(): """ 创建日志文件 """ fields = ['datetime', 'RSSI', 'SNR', 'msg'] dir_path = os.path.join(root_path, 'lora_data') os.makedirs(dir_path, exist_ok=True) now = datetime.now() file_name = now.strftime('%Y%m%d') + '_rx.csv' file_path = os.path.join(dir_path, file_name) if not os.path.exists(file_path): with open(file_path, 'w') as fw: fw.write(','.join(fields) + '\n') return file_path
def fit(self,trainFile=None,with_validation=general_config.with_validation, log_dir=general_config.log_dir+"/TextCNN",save_dir=general_config.save_dir+"/TextCNN", load_path=general_config.load_path_train, num_epochs=general_config.num_epochs, steps_every_epoch=general_config.steps_every_epoch, batch_size=general_config.batch_size, learning_rate=general_config.learning_rate, lr_changing=general_config.lr_changing, min_learning_rate=general_config.min_learning_rate, learning_rate_decay=general_config.learning_rate_decay, save_epochs=general_config.save_epochs, early_stopping=general_config.early_stopping, num_visual=general_config.num_visualize): self.learning_rate_value = learning_rate self.trainFile = trainFile self.validFile = None self.with_validation = with_validation if self.trainFile is None: if self.with_validation: self.trainFile = general_config.train_file else: self.trainFile = general_config.training_file if self.with_validation: self.validFile = self.trainFile.replace("train", "valid") tmp = os.path.join(os.path.dirname(self.trainFile), os.path.basename(self.trainFile).replace(".txt", "").split("_")[0]) if self.model_type in ["static","multichannel"]: self.int2vocabPath = general_config.global_static_i2v_path self.vocab2intPath = general_config.global_static_v2i_path else: self.int2vocabPath = tmp + "_i2v.json" self.vocab2intPath = tmp + "_v2i.json" metadataPath = { "static": "/home/leechen/code/python/TextSentimentClassification/data_helpers/dataset/training_testing_metadata.tsv"} metadataPath["nonstatic"] = "/home/leechen/code/python/TextSentimentClassification/" \ + self.vocab2intPath.replace("v2i.json", "metadata.tsv") train_loss = [] train_accuracy = [] valid_loss = [] valid_accuracy = [] # 训练过程中的日志保存文件以及模型保存路径 if self.with_validation: log_dir=ensure_dir_exist(log_dir+"/"+self.model_type+"/train_valid") train_dir = os.path.join(log_dir, "train") val_dir = os.path.join(log_dir, "valid") save_dir = ensure_dir_exist(save_dir + "/" + self.model_type + "/train_valid") else: log_dir=ensure_dir_exist(log_dir+"/"+self.model_type+"/train") train_dir = os.path.join(log_dir, "train") val_dir=None save_dir = ensure_dir_exist(save_dir + "/" + self.model_type + "/train") # 生成日志 logger=my_logger(log_dir+"/log_fit.txt") msg = "\n--filter_size_list: %s\n" % self.filter_size_list \ + "--filter_num: %s\n" % self.filter_num \ + "--fc_layer_size_list: %s\n" % self.fc_layer_size_list \ + "--embedding_size: %s\n" % self.embedding_size \ + "--dropout: %s\n" % self.dropout_value \ + "--max_l2_norm: %s\n" % self.max_l2_norm \ + "--learning_rate: %s\n" % self.learning_rate_value \ + "--lr_changing: %s\n" % lr_changing \ + "--min_learning_rate: %s\n" % min_learning_rate\ + "--learning_rate_decay: %s\n" % learning_rate_decay\ +"--load_path: %s\n" % load_path\ +"--num_epochs: %s\n" % num_epochs\ +"--steps_every_epoch: %s\n" % steps_every_epoch\ +"--batch_size: %s\n" % batch_size\ +"--save_epochs: %s\n" % save_epochs\ +"--early_stopping: %s\n" % early_stopping\ +"--num_visual: %s\n"%num_visual logger.info(msg) # 定义数据生成器 train_generator = PaddedDataIterator(loadPath=self.trainFile,vocab2intPath=self.vocab2intPath) val_generator = None if self.validFile is None else PaddedDataIterator(loadPath=self.validFile, vocab2intPath=self.vocab2intPath) os.environ["CUDA_VISIBLE_DEVICES"] = str(0) config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.8 with tf.Session(config=config,graph=self.graph) as sess: train_writer = tf.summary.FileWriter(train_dir, sess.graph) val_writer = None if val_dir is None else tf.summary.FileWriter(val_dir) saver = tf.train.Saver(max_to_keep=5) sess.run(tf.global_variables_initializer()) start = 0 if isinstance(load_path,str): if os.path.isdir(load_path): ckpt = tf.train.get_checkpoint_state(load_path) saver.restore(sess, ckpt.model_checkpoint_path) start = ckpt.model_checkpoint_path.split("-")[-1] else: saver.restore(sess, load_path) start = load_path.split("-")[-1] logger.info("Loading successfully, loading epoch is %s" % start) logger.info("The total number of trainable variables: %s"%get_num_params()) cur_early_stopping=0 cur_max_acc=0. logger.info('******* start training with %d *******' % start) epoch=0 for epoch in range(start, num_epochs): if lr_changing: try: if (train_loss[-1]>train_loss[-2]): tmp=self.learning_rate_value*learning_rate_decay if (tmp>=min_learning_rate): self.learning_rate_value=tmp logger.info("Learning rate multiplied by %s at epoch %s." %(learning_rate_decay,epoch+1)) else: if (train_loss[-1]<train_loss[-2]-0.015): self.learning_rate_value*=1.05 logger.info("Learning rate multiplied by 1.05 at epoch %s."%(epoch+1)) except: pass avg_loss_t, avg_accuracy_t = 0, 0 avg_loss_v, avg_accuracy_v = 0, 0 for step in range(steps_every_epoch): _, batch_seqs, batch_labels, _ = train_generator.next(batch_size) batch_seqs_ns=None if self.model_type=="multichannel": batch_seqs_ns = self._X2X_ns(batch_seqs) sess.run(self.train_op,self._feed_dict_train(batch_x=batch_seqs, batch_y=batch_labels, batch_x_ns=batch_seqs_ns)) loss_t, acc_t= sess.run([self.loss_op, self.acc_op], self._feed_dict_valid(batch_x=batch_seqs, batch_y=batch_labels, batch_x_ns=batch_seqs_ns)) avg_loss_t += loss_t avg_accuracy_t += acc_t avg_loss_t/=steps_every_epoch avg_accuracy_t/=steps_every_epoch train_loss.append(avg_loss_t) train_accuracy.append(avg_accuracy_t) self.loss_accuracy_summary.value[0].simple_value = avg_loss_t self.loss_accuracy_summary.value[1].simple_value = avg_accuracy_t train_writer.add_summary(summary=self.loss_accuracy_summary, global_step=epoch + 1) if self.with_validation: # 计算验证集上的表现 cur_loop=val_generator.loop _, batch_seqs, batch_labels,_ = val_generator.next(1024,need_all=True) batch_seqs_ns=None if self.model_type == "multichannel": batch_seqs_ns = self._X2X_ns(batch_seqs) cur_count=0 while(val_generator.loop==cur_loop): loss_v, acc_v = sess.run([self.loss_op, self.acc_op], feed_dict= self._feed_dict_valid(batch_x=batch_seqs, batch_y=batch_labels,batch_x_ns=batch_seqs_ns)) avg_loss_v += loss_v avg_accuracy_v += acc_v cur_count += 1 _, batch_seqs, batch_labels, _ = val_generator.next(1024, need_all=True) batch_seqs_ns = None if self.model_type == "multichannel": batch_seqs_ns = self._X2X_ns(batch_seqs) avg_loss_v/=cur_count avg_accuracy_v/=cur_count valid_loss.append(avg_loss_v) valid_accuracy.append(avg_accuracy_v) self.loss_accuracy_summary.value[0].simple_value = avg_loss_v self.loss_accuracy_summary.value[1].simple_value = avg_accuracy_v val_writer.add_summary(summary=self.loss_accuracy_summary, global_step=epoch + 1) logger.info("Epoch: [%04d/%04d], " "Training Loss: %.4f, Training Accuracy: %.4f, " "Validation Loss: %.4f, Validation Accuracy: %.4f" \ % (epoch + 1, num_epochs, avg_loss_t, avg_accuracy_t, avg_loss_v, avg_accuracy_v)) # 如果验证集上的准确率连续低于历史最高准确率的次数超过early_stopping次,则提前停止迭代。 if (avg_accuracy_v > cur_max_acc): cur_max_acc = avg_accuracy_v cur_early_stopping = 0 logger.info("Saving model-%s" % (epoch + 1)) saver.save(sess, os.path.join(save_dir, 'model.ckpt'), global_step=epoch + 1) else: cur_early_stopping += 1 if cur_early_stopping > early_stopping: logger.info("Early stopping after epoch %s !" % (epoch + 1)) break else: logger.info("Epoch: [%04d/%04d], " "Training Loss: %.4f, Training Accuracy: %.4f " \ % (epoch + 1, num_epochs,avg_loss_t, avg_accuracy_t)) # 保存一次模型 if (epoch - start + 1) % save_epochs == 0: logger.info("Saving model-%s"%(epoch+1)) saver.save(sess, os.path.join(save_dir, 'model.ckpt'), global_step=epoch + 1) if num_visual > 0: # 可视化最终词向量 config = projector.ProjectorConfig() final_embeddings = {} try: final_embeddings["static"] = self.embedding_matrix_s.eval()[:num_visual] except: pass try: final_embeddings["nonstatic"] = self.embedding_matrix_ns.eval()[:num_visual] except: pass for (name, final_embedding) in final_embeddings.items(): embedding_var = tf.Variable(final_embedding, name="word_embeddings_" + name) sess.run(embedding_var.initializer) saver = tf.train.Saver([embedding_var]) saver.save(sess, log_dir + "/embeddings_" + name+".ckpt-"+str(epoch+1)) embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = metadataPath[name] projector.visualize_embeddings(train_writer, config) return train_loss, train_accuracy, valid_loss, valid_accuracy
from time import sleep import pigpio import pynmea2 from utils import my_logger logger = my_logger(__name__) class UP501(object): PPS = 22 RXD = 14 TXD = 15 def __init__(self, pi): self.pi = pi self.pi.set_mode(UP501.PPS, pigpio.INPUT) self.pi.set_mode(UP501.TXD, pigpio.INPUT) self.pi.set_mode(UP501.RXD, pigpio.OUTPUT) def read(self): status = self.pi.bb_serial_read_open(UP501.TXD, 9600, 8) while True: (count, data) = self.pi.bb_serial_read(UP501.TXD) sleep(1) if count > 0: break self.pi.bb_serial_read_close(UP501.TXD) res = '' try: res = data.decode() except: pass
def run(): logger = my_logger() get_url = settings.URL + '?vm_audit=1&vm_installed=0' res = get_args(get_url) install_li = res.get('results') logger.debug(install_li) if install_li: for item in install_li: starting = True try: if item["room_name"] == 'yi': vm_obj = AutoVM(settings.FVCENTER_USER, settings.FVCENTER_PASSWORD, settings.FVCENTER_IP, item["room_name"], item["host_ip"], item["datastore"], item["vm_ip"], item["vm_gateway"], item["vm_cpu"], item["vm_memory"], item["vm_disk"], item["vm_name"], item["vm_proposer"], item["id"]) elif item["room_name"] == 'two': vm_obj = AutoVM(settings.TVCENTER_USER, settings.TVCENTER_PASSWORD, settings.TVCENTER_IP, item["room_name"], item["host_ip"], item["datastore"], item["vm_ip"], item["vm_gateway"], item["vm_cpu"], item["vm_memory"], item["vm_disk"], item["vm_name"], item["vm_proposer"], item["id"]) elif item["room_name"] == 'san': vm_obj = AutoVM(settings.SVCENTER_USER, settings.SVCENTER_PASSWORD, settings.SVCENTER_IP, item["room_name"], item["host_ip"], item["datastore"], item["vm_ip"], item["vm_gateway"], item["vm_cpu"], item["vm_memory"], item["vm_disk"], item["vm_name"], item["vm_proposer"], item["id"]) else: raise Exception('机房名称错误') # 创建虚拟机 vm_obj.create_VM() # 启用cpu mem热插拔 vm_obj.hot_plug() # 打包镜像 vm_obj.iso_pac() # 上传镜像 up = vm_obj.upload_iso(settings.ISO_LOCAL) if up: # 开机自动安装 vm_obj.power_on() # 设置api为已安装 patch_url = settings.URL + '%s/' % item["id"] patch_installed(patch_url) # # 判断启动成功并断开cdrom # count=0 # while starting and count<34: # on=judgment_on(item["vm_ip"]) # if on: # vm_obj.connect_cdrom(connect=False) # starting=False # else: # time.sleep(10) # count += 1 logger.info('Successful 机房:%s 虚拟机:%s 装机中...请稍后再试' % (item["room_name"], item["vm_name"])) except Exception as e: logger.error('机房:%s 虚拟机:%s 装机失败,请查看失败原因并手动删除主机 %s' % (item["room_name"], item["vm_name"], e)) continue
results.add("mwm_jaccard", metrics_cls["maximum_weighted_match_jaccard"]) results.add("inst-precision", metrics_cls["pair_precision"]) results.add("inst-recall", metrics_cls["pair_recall"]) results.add("inst-f1", metrics_cls["pair_f1"]) return results if __name__ == '__main__': args = read_options() # Add TensorBoard Writer writer = SummaryWriter(log_dir=None, comment=args.comment) # Add Python Logger logger = my_logger(name='exp', log_path=writer.file_writer.get_logdir()) logger.setLevel(0) # Save Parameters in TensorBoard Writer and Logger options = vars(args) logger.info("Command Line Options: {}".format(options)) writer.add_text('Text', 'Hyper-parameters: {}'.format(options), 0) # Initialize random seed random.seed(args.random_seed) torch.manual_seed(args.random_seed) np.random.seed(args.random_seed) if args.device_id != -1: torch.cuda.manual_seed_all(args.random_seed) torch.backends.cudnn.deterministic = True torch.set_printoptions(precision=9)