def start(instance="default"): """ Attach ipmi-console to target instance specified by its name :param instance: infrasim instance name """ # initialize logging global logger_ic logger_ic = infrasim_log.get_logger(LoggerType.ipmi_console.value, instance) common.init_logger(instance) # initialize environment env.local_env.quit_flag = False common.init_env(instance) pid_file = "{}/{}/.ipmi_console.pid".format(config.infrasim_home, instance) daemon.daemonize(pid_file) with open(pid_file, "r") as fp: logger_ic.info("ipmi-console of {} start with pid {}". format(instance, fp.read().strip())) # parse the sdrs and build all sensors sdr.parse_sdrs() # running thread for each threshold based sensor _start_monitor(instance) _spawn_sensor_thread() _start_console(instance)
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument("--do_train", default=True, action='store_true') parser.add_argument('--do_eval', default=False, action='store_true') parser.add_argument("--do_predict", default=False, action='store_true') parser.add_argument('--markup', default='bieos', type=str, choices=['bios', 'bio', 'bieos']) # 标签类型 parser.add_argument("--arch", default='bilstm_crf', type=str) parser.add_argument('--learning_rate', default=0.001, type=float) parser.add_argument('--seed', default=1234, type=int) parser.add_argument('--gpu', default='', type=str) parser.add_argument('--epochs', default=100, type=int) parser.add_argument('--batch_size', default=32, type=int) parser.add_argument('--embedding_size', default=128, type=int) parser.add_argument('--hidden_size', default=384, type=int) parser.add_argument("--grad_norm", default=5.0, type=float, help="Max gradient norm.") parser.add_argument("--task_name", type=str, default='ner') args = parser.parse_args() args.data_dir = config.data_dir if not config.output_dir.exists(): args.output_dir.mkdir() args.output_dir = config.output_dir / '{}'.format(args.arch) if not args.output_dir.exists(): args.output_dir.mkdir() init_logger(log_file=str(args.output_dir / '{}-{}.log'.format(args.arch, args.task_name))) seed_everything(args.seed) if args.gpu != '': args.device = torch.device(f"cuda:{args.gpu}") else: args.device = torch.device("cpu") args.id2label = {i: label for i, label in enumerate(config.label2id)} args.label2id = config.label2id processor = CluenerProcessor(data_dir=config.data_dir) processor.get_vocab() model = BERT_NERModel(device=args.device, label2id=args.label2id, need_birnn=True) # model = NERModel(vocab_size=len(processor.vocab), embedding_size=args.embedding_size, # hidden_size=args.hidden_size, device=args.device, label2id=args.label2id) model.to(args.device) if args.do_train: train(args, model, processor) if args.do_eval: model_path = args.output_dir / 'best-model.bin' model = load_model(model, model_path=str(model_path)) evaluate(args, model, processor) if args.do_predict: predict(args, model, processor)
def start(instance="default"): """ Attach ipmi-console to target instance specified by its name :param instance: infrasim instance name """ # initialize logging global logger_ic logger_ic = infrasim_log.get_logger(LoggerType.ipmi_console.value, instance) common.init_logger(instance) # initialize environment env.local_env.quit_flag = False common.init_env(instance) daemon.daemonize("{}/{}/.ipmi_console.pid".format(config.infrasim_home, instance)) # parse the sdrs and build all sensors sdr.parse_sdrs() # running thread for each threshold based sensor _start_monitor(instance) _spawn_sensor_thread() _start_console(instance)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--do_train", default=False, action='store_true') parser.add_argument("--do_eval", default=False, action='store_true') parser.add_argument('--do_predict', default=False, action='store_true') #bios,s是single单字 parser.add_argument('--markup', default='bios', type=str, choices=['bio', 'bios']) parser.add_argument("--arch", default='bert', type=str) parser.add_argument('--seed', default=1234, type=int) parser.add_argument('--gpu', default='0', type=str) parser.add_argument('--epochs', default=20, type=int) parser.add_argument('--batch_size', default=16, type=int) parser.add_argument("--task_name", type=str, default='ner') parser.add_argument("--max_seq_len", type=int, default=64) parser.add_argument("--aug_num", default=4, type=int) args = parser.parse_args() args.vocab_file = config.vocab_file args.data_dir = config.data_dir args.tf_checkpoint = config.tf_checkpoint args.init_checkpoint = config.init_checkpoint args.bert_config_file = config.bert_config_file if not config.output_dir.exists(): args.output_dir.mkdir() args.output_dir = config.output_dir / '{}'.format(args.arch) if not args.output_dir.exists(): args.output_dir.mkdir() init_logger(log_file=str(args.output_dir / '{}-{}.log'.format(args.arch, args.task_name))) seed_everything(args.seed) #设置gpu os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu args.id2label = {i: label for i, label in enumerate(config.label2id)} args.label2id = config.label2id processor = CluenerProcessor(data_dir=config.data_dir, vocab_file=config.vocab_file, label2id=config.label2id, max_seq_len=args.max_seq_len) processor.get_label_embedding(args.label2id, config.pretrain_label_embedding_file, config.label_embedding_file) args.label_embedding = processor.label_embedding if args.do_train: train(args, NERModel, processor) if args.do_predict: predict(args, processor)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--do_train", default=False, action='store_true') parser.add_argument("--do_eval", default=False, action='store_true') parser.add_argument('--do_predict', default=False, action='store_true') parser.add_argument('--markup', default='bmeso', type=str) parser.add_argument("--arch", default='transformer', type=str) parser.add_argument('--learning_rate', default=0.001, type=float) parser.add_argument('--seed', default=1234, type=int) parser.add_argument('--gpu', default='0', type=str) parser.add_argument('--epochs', default=100, type=int) parser.add_argument('--batch_size', default=16, type=int) # parser.add_argument('--hidden_size', default=512, type=int) parser.add_argument("--grad_norm", default=5.0, type=float, help="Max gradient norm.") args = parser.parse_args() args.data_dir = config.data_dir args.output_dir = config.output_dir / Path( '{}'.format(str(config.dataset) + '-on-lstm')) if not args.output_dir.exists(): args.output_dir.mkdir() init_logger(log_file=str(args.output_dir / '{}.log'.format(args.arch))) seed_everything(args.seed) # 设置gpu os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if os.path.exists(config.save_data_name): print('Loading processed data') with open(config.save_data_name, 'rb') as fp: data = pickle.load(fp) else: data = Data() data_initialization(data, config.gaz_file, config.train_path, config.dev_path, config.test_path) data.generate_instance_with_gaz(config.train_path, 'train') data.generate_instance_with_gaz(config.dev_path, 'dev') data.generate_instance_with_gaz(config.test_path, 'test') data.build_word_pretrain_emb(config.char_emb) data.build_biword_pretrain_emb(config.bichar_emb) data.build_gaz_pretrain_emb(config.gaz_file) print('Dumping data') with open(config.save_data_name, 'wb') as f: pickle.dump(data, f) if args.do_train: train(args, data, NERModel) if args.do_predict: predict(args, data, NERModel, 'test')
def __init__(self, vocab_size, dim_emb=DIM_EMB, *args, **kwargs): """Initialize a network.""" super(CBOW, self).__init__() self.verbose = kwargs.get('verbose', False) self.logger = init_logger('CBOW') self.embeddings_x = nn.Embedding(vocab_size, dim_emb, sparse=True) self.embeddings_y = nn.Embedding(vocab_size, dim_emb, sparse=True)
def main(): # 注册退出回调函数 #atexit.register(main_shutdown) logger = common.init_logger("server") proxy = Proxy(logger) proxy.start() proxy.join() logger.info('server end')
def main(): parser = argparse.ArgumentParser() parser.add_argument("--do_train", default=False, action='store_true') parser.add_argument("--do_eval", default=False, action='store_true') parser.add_argument('--do_predict', default=False, action='store_true') #bios,s是single单字 parser.add_argument('--markup', default='bios', type=str, choices=['bio', 'bios']) parser.add_argument("--arch", default='bilstm_crf', type=str) parser.add_argument('--learning_rate', default=0.001, type=float) parser.add_argument('--seed', default=1234, type=int) parser.add_argument('--gpu', default='0', type=str) parser.add_argument('--epochs', default=80, type=int) parser.add_argument('--batch_size', default=32, type=int) parser.add_argument('--embedding_size', default=128, type=int) parser.add_argument('--hidden_size', default=384, type=int) parser.add_argument("--grad_norm", default=5.0, type=float, help="Max gradient norm.") parser.add_argument("--task_name", type=str, default='ner') args = parser.parse_args() args.data_dir = config.data_dir if not config.output_dir.exists(): args.output_dir.mkdir() args.output_dir = config.output_dir / '{}'.format(args.arch) if not args.output_dir.exists(): args.output_dir.mkdir() init_logger(log_file=str(args.output_dir / '{}-{}.log'.format(args.arch, args.task_name))) seed_everything(args.seed) #设置gpu os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu args.id2label = {i: label for i, label in enumerate(config.label2id)} args.label2id = config.label2id processor = CluenerProcessor(data_dir=config.data_dir) processor.get_vocab() if args.do_train: train(args, NERModel, processor) if args.do_predict: predict(args, NERModel, processor)
def __init__(self): self.logger = common.init_logger("client") self.logger.info("test") _addr, _port = config.proxy.split(':') self.proxy_port = int(_port) self.proxy_addr = _addr self.my_name = config.user self.registered = False self.url_parser = re.compile("((.*):)?(\d+)")
def __init__(self): self.logger = common.init_logger("client") self.logger.info("test") yaml_pathname = common.build_filename("client.yaml") f = open(yaml_pathname) cfg = yaml.load(f) f.close() _addr, _port = cfg['proxy'].split(':') self.proxy_port = int(_port) self.proxy_addr = _addr self.my_name = cfg['user'] self.registered = False self.url_parser = re.compile("((.*):)?(\d+)")
def __init__(self, window_size, *args, **kwargs): "Set variables." self.verbose = kwargs.get('verbose', False) self.logger = init_logger('Corpus') self.window_size = window_size # length of window on each side # Vocabulary self.w2i = defaultdict(lambda: len(self.w2i)) self.freq = defaultdict(int) self.UNK = self.w2i['<unk>'] self.BOS = self.w2i['<s>'] self.EOS = self.w2i['</s>'] self.freq[self.UNK] = 0 self.freq[self.BOS] = 0 self.freq[self.EOS] = 0 self.vocab_range = {}
'docker rm -fv $(docker ps -aq --filter=label=com.avatao.typed_crp_id=docker)' ], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL).wait() if __name__ == '__main__': """ Run avatao solvable and controller docker images. Simply add the challenge repository path as the first argument and the script does the rest. Python dependencies: - PyYAML (http://pyyaml.org/) or simply `pip3 install PyYAML` (on Ubuntu you additionally need `apt-get install python3-yaml`) """ init_logger() repo_path, repo_name = get_sys_args() os.chdir(repo_path) atexit.register(remove_containers) proc_list = [] first = None if 'crp_config' not in read_config(repo_path): logging.warning('There is no crp_config in the config.yml, if this is ' 'a static challenge you don\'t need to run it.') sys.exit(0) for short_name, crp_config_item in get_crp_config(repo_path, repo_name).items():
for sensor_obj in sdr.sensor_list: sensor_obj.set_mode("user") # set quit flag sensor_obj.set_quit(True) # acquire the lock that before notify sensor_obj.condition.acquire() sensor_obj.condition.notify() sensor_obj.condition.release() for thread in sensor_thread_list: thread.join() if __name__ == '__main__': # initialize logging common.init_logger() # open telnet session to IPMI simulator # common.open_telnet_session() # parse the sdrs and build all sensors sdr.parse_sdrs() # running thread for each threshold based sensor spawn_sensor_thread() if SERVERTYPE == 'threaded': # Single threaded server - only one session at a time class TelnetServer(SocketServer.TCPServer): allow_reuse_address = True server = TestSSHHandler(port=SSH_PORT_BINDING)
#!/usr/bin/env python import os import re import common import regexs from collections import defaultdict from operator import itemgetter import signal import multiprocessing import argparse g_logger = common.init_logger() parser = argparse.ArgumentParser() parser.add_argument('-d', '--dir', type=str, default='./', help='the log file diretory.') parser.add_argument('-p', '--prev', action='store_true', help='if parse prev logs.') parser.add_argument('-f', '--status_file', action='store_true', help='if get path from the local file.') args = parser.parse_args() g_shared_list = multiprocessing.Manager().list()
""" import os import sys import datetime import time import traceback home_dir = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(home_dir + "/bin") sys.path.append(home_dir + "/conf") import common import etl import make_train_data import common log = common.init_logger() def main(): """主函数""" stocks = common.load_all_stocks() for i, stock in enumerate(stocks): try: # if i < 2809: # continue stock_no = stock['stock_no'] print stock_no start = stock['start'] log.info("run %s,%s,%s" %(stock_no,start,i)) # etl.download(stock_no,start) #下载
sensor_obj.set_mode("user") # set quit flag sensor_obj.set_quit(True) # acquire the lock that before notify sensor_obj.condition.acquire() sensor_obj.condition.notify() sensor_obj.condition.release() for thread in sensor_thread_list: thread.join() if __name__ == '__main__': # initialize logging common.init_logger() # open telnet session to IPMI simulator # common.open_telnet_session() # parse the sdrs and build all sensors sdr.parse_sdrs() # running thread for each threshold based sensor spawn_sensor_thread() if SERVERTYPE == 'threaded': # Single threaded server - only one session at a time class TelnetServer(SocketServer.TCPServer): allow_reuse_address = True server = TestSSHHandler(port=SSH_PORT_BINDING)
close_and_swallow(pipefd[1]) logger.debug("wait_on_haproxy_pipe done (False)") return False except OSError as e: logger.debug("wait_on_haproxy_pipe OSError: %s", e) if e.args[0] != errno.EINTR: close_and_swallow(pipefd[0]) close_and_swallow(pipefd[1]) logger.debug("wait_on_haproxy_pipe done (False)") return False logger.debug("wait_on_haproxy_pipe done (True)") return True init_logger( "/dev/null", "%(asctime)s.%(msecs)03d%(timezoneiso8601) %(levelname)s - 0 python %(name)s {\"@message\": \"%(message)s\"}", "DEBUG") logger = common.marathon_lb_logger.getChild('haproxy_wrapper.py') pipefd = create_haproxy_pipe() pid = os.fork() if not pid: os.environ["HAPROXY_WRAPPER_FD"] = str(pipefd[1]) # Close the read side os.close(pipefd[0]) os.execv(sys.argv[1], sys.argv[1:]) # Close the write side os.close(pipefd[1])
#!/usr/bin/env python import time import boto.ec2 import re import sys import os import logging import socket from common import load_config, get_aws_public_hostname, init_logger init_logger() from fabric.api import * # http://examples.oreilly.com/0636920020202/ec2_launch_instance.py def connect(): config = load_config() #ConfigParser.ConfigParser() aws_key = config.get('aws', 'aws_key') aws_secret = config.get('aws', 'aws_secret') conn = boto.ec2.connect_to_region( 'us-east-1', aws_access_key_id = aws_key, aws_secret_access_key = aws_secret, ) return conn
print('[{}] loss = {:.4f} ({:.4f}/{:.4f}), time = {:.2f}'.format( ITER + 1, train_loss_we + train_loss_d, train_loss_we, train_loss_d, time.time() - start)) # Save vectors embs = model.get_embeddings() save_embeddings(args.path_output, embs, corpus.i2w) if args.dir_model: save_model(args.dir_model, model, corpus.w2i) return 0 if __name__ == '__main__': logger = init_logger('MAIN') parser = argparse.ArgumentParser() parser.add_argument('-s', '--src', dest='path_src', required=True, help='path to a source corpus file') parser.add_argument('-t', '--trg', dest='path_trg', required=True, help='path to a target corpus file') parser.add_argument('--window-size', type=int, default=2, help='window size on each side')
help="The proxy listen port.") tmpl_parser = subparsers.add_parser('tmpl', help="Use default template.") tmpl_parser.add_argument("title", type=str, help="Web site title.") tmpl_parser.set_defaults(func=run_tmpl) clone_parser = subparsers.add_parser('clone', help="Clone web site from url.") clone_parser.add_argument("url", type=str, help="Url to clone.") clone_parser.add_argument("--path", type=str, default="login", help="Request path to be logged.") clone_parser.add_argument("--ext", type=str, help="Extends (.py).") clone_parser.set_defaults(func=run_clone) spec_parser = subparsers.add_parser('spec', help="Use specified web site file.") spec_parser.add_argument("wwwroot", type=str, help="Web site html file.") spec_parser.add_argument("--path", type=str, default="login", help="Request path to be logged.") spec_parser.set_defaults(func=run_spec) args = parser.parse_args() init_logger("fakelogin.log", args.debug) args.func(args) IPython.embed()
except OSError as e: logger.debug("wait_on_haproxy_pipe OSError: %s", e) if e.args[0] != errno.EINTR: close_and_swallow(pipefd[0]) close_and_swallow(pipefd[1]) logger.debug("wait_on_haproxy_pipe done (False)") return False logger.debug("wait_on_haproxy_pipe done (True)") return True syslog_socket = os.getenv("WRAPPER_SYSLOG_SOCKET") log_format = os.getenv("WRAPPER_LOG_FORMAT") log_level = os.getenv('WRAPPER_LOG_LEVEL') init_logger(syslog_socket, log_format, log_level) logger = common.marathon_lb_logger.getChild('haproxy_wrapper.py') pipefd = create_haproxy_pipe() pid = os.fork() if not pid: os.environ["HAPROXY_WRAPPER_FD"] = str(pipefd[1]) # Close the read side os.close(pipefd[0]) os.execv(sys.argv[1], sys.argv[1:]) # Close the write side os.close(pipefd[1]) while wait_on_haproxy_pipe(pipefd):
tw_in = sum(tw[:, 1] < 0) features.extend([tw_out, tw_in]) return features return new_x def build_dict(features): vocab, inverse = np.unique(features, return_inverse=True) return vocab, inverse.reshape(features.shape[0], features.shape[1]) if __name__ == '__main__': global n, w '''initialize logger''' logger = common.init_logger("extract") '''read config file''' cf = common.read_conf(common.confdir) MON_SITE_NUM = int(cf['monitored_site_num']) MON_INST_NUM = int(cf['monitored_inst_num']) num_class = MON_SITE_NUM if cf['open_world'] == '1': UNMON_SITE_NUM = int(cf['unmonitored_site_num']) num_class += 1 else: UNMON_SITE_NUM = 0 '''read in arg''' parser = argparse.ArgumentParser(description='DF feature extraction') parser.add_argument( '--dir', metavar='<traces path>',