def test_create_logger(): path1 = os.path.join(TMP_DIR, "1.log") path2 = os.path.join(TMP_DIR, "2.log") log1 = log.create_logger("LogOne", lvl="DEBUG", path=path1) log2 = log.create_logger("LogTwo", lvl="INFO", path=path2) log3 = log.create_logger("LogThree", lvl="ERROR", path=path2) log_levels(log1) log_levels(log2) log_levels(log3) with open(path1) as file1: lines = file1.readlines() assert len(lines) == len(EXPECTED_LOGS) for line, expected_log in zip(lines, EXPECTED_LOGS): assert expected_log.format("LogOne") in line with open(path2) as file2: lines = file2.readlines() assert len(lines) == 4 + 2 for line, expected_log in zip(lines[:4], EXPECTED_LOGS[-4:]): assert expected_log.format("LogTwo") in line for line, expected_log in zip(lines[4:], EXPECTED_LOGS[-2:]): assert expected_log.format("LogThree") in line
def start_test(self, artifact_path, test_name, test_location, slaveid): if not slaveid: slaveid = "Master" test_ident = "{}/{}".format(test_location, test_name) if slaveid in self.store: if self.store[slaveid].in_progress: print "Test already running, can't start another, logger" return None self.store[slaveid] = self.Test(test_ident) self.store[slaveid].in_progress = True artifacts = [] os_filename = self.ident + "-" + "cfme.log" os_filename = os.path.join(artifact_path, os_filename) if os.path.isfile(os_filename): os.remove(os_filename) artifacts.append(os_filename) self.store[slaveid].logger = create_logger(self.ident + test_name, os_filename) self.store[slaveid].logger.setLevel(self.level) for log_name in artifacts: desc = log_name.rsplit("-", 1)[-1] self.fire_hook('filedump', test_location=test_location, test_name=test_name, description=desc, slaveid=slaveid, contents="", file_type="log", display_glyph="align-justify", dont_write=True, os_filename=log_name, group_id="pytest-logfile")
def main2(args): print "Running..." # get extracted wikipedia file pathnames subdirs = io.list_directories(config.WIKIPEDIA_EXTRACTED_DIR) if args.letters: subdirs = [p for p in subdirs if p[-2] in config.WIKIPEDIA_SUB_DIR_PREFIXES] pathnames = [] for sb in subdirs: pathnames.extend(io.list_files(sb)) pathnames.sort() # create thread-safe queue queue = parallel.create_queue(pathnames) # create workers workers = [] for i in xrange(args.threads): logger = log.create_logger('LOGGER %d' % i, 'log_%d.log' % i) sent_tokenizer = parser.get_sentence_tokenizer() if args.verbose: logger.setLevel(logging.DEBUG) worker = SentenceIndexWorker(queue, sent_tokenizer, logger) workers.append(worker) # begin for worker in workers: worker.start() # block until all files have been processed queue.join() print "Done!"
def start_test(self, artifact_path, test_name, test_location, slaveid): if not slaveid: slaveid = "Master" test_ident = "{}/{}".format(test_location, test_name) if slaveid in self.store: if self.store[slaveid].in_progress: print("Test already running, can't start another, logger") return None self.store[slaveid] = self.Test(test_ident) self.store[slaveid].in_progress = True artifacts = [] os_filename = self.ident + "-" + "cfme.log" os_filename = os.path.join(artifact_path, os_filename) if os.path.isfile(os_filename): os.remove(os_filename) artifacts.append(os_filename) self.store[slaveid].logger = create_logger(self.ident + test_name, os_filename) self.store[slaveid].logger.setLevel(self.level) for log_name in artifacts: desc = log_name.rsplit("-", 1)[-1] self.fire_hook('filedump', test_location=test_location, test_name=test_name, description=desc, slaveid=slaveid, contents="", file_type="log", display_glyph="align-justify", dont_write=True, os_filename=log_name, group_id="pytest-logfile")
def start_test(self, artifact_path, test_name, test_location, slaveid): if not slaveid: slaveid = "Master" test_ident = "{}/{}".format(test_location, test_name) if slaveid in self.tests: if self.tests[slaveid].in_progress: print "Test already running, can't start another" return None self.tests[slaveid] = self.Test(slaveid) self.tests[slaveid].in_progress = True artifacts = [] os_filename = self.ident + "-" + "cfme.log" os_filename = os.path.join(artifact_path, os_filename) if os.path.isfile(os_filename): os.remove(os_filename) artifacts.append(os_filename) self.tests[slaveid].logger = create_logger(self.ident + test_name, os_filename) self.tests[slaveid].logger.setLevel(self.level) return None, { 'artifacts': { test_ident: { 'files': { self.ident: artifacts } } } }
def __init__(self, game_cls, params={}): """Create an agent given a game class. Args: params (dict): discount_rate default is 0.95 learning_rate default is 0.001 exploration_rate default is 1 exploration_decay default is 0.995 exploration_min default is 0.01 """ self.number = Agent.agents_n Agent.agents_n += 1 self.states = game_cls.states self.actions = game_cls.actions self.action_size = len(self.actions) self.qvalues = [ [0 for action in self.actions] for state in self.states ] self.extras = { 'cumul_reward': 0, 'updates_n': 0, } self.history = History(10) self.params = dict(Agent.params) self.set_params(**params) self.log = create_logger(str(self), log_level=LOG_LEVEL) self.log.info("Agent initialized with params: %s.", self.params)
def __init__(self, cmd): """ Args: cmd: command line parameters. """ self.log = create_logger( __name__, silent=False, to_disk=True, log_file=cfg.log) self.cmd = cmd self.log.info(str(cmd))
def create_app(config_type): """应用初始化""" # 创建Flask应用 app = create_flask_app(config_type) # 初始化组件 register_extensions(app) # 添加日志 app.config['PROPAGATE_EXCEPTIONS'] = False #设置传播异常 from utils.log import create_logger create_logger(app) from utils.middlewares import get_userinfo app.before_request(get_userinfo) # 注册蓝图 register_blueprint(app) return app
def __init__(self): super().__init__() self.board = [] for row in range(3): for col in range(3): self.board.append(Cell(row, col)) self.history = defaultdict(list) self.winner = None self.ended = False self.players = [Player(0, "X"), Player(1, "O")] self.log = create_logger(self.__class__.__name__, log_level=LOG_LEVEL)
def handle(self): """ Handle multiple requests - each expected to be a 4-byte length, followed by the LogRecord in pickle format. Logs the record according to whatever policy is configured locally. """ global logger_cache while True: chunk = self.connection.recv(4) if len(chunk) < 4: break slen = struct.unpack(">L", chunk)[0] chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + self.connection.recv(slen - len(chunk)) obj = pickle.loads(chunk) record = logging.makeLogRecord(obj) # if a name is specified, we use the named logger rather than the one # implied by the record. if self.server.logname is not None: name = self.server.logname else: name = record.name if not name: filename = logs_path.join("sprout.log") else: fields = name.split(".") fields[-1] += ".log" filename = logs_path for field in fields: filename = filename.join(field) if not field.endswith(".log"): with global_fs_lock: if not filename.exists(): filename.mkdir() filename = filename.strpath with logger_cache_lock: if filename in logger_cache: logger, lock = logger_cache[filename] else: logger = create_logger(name, filename=filename, max_file_size=MAX_FILE_SIZE, max_backups=MAX_BACKUPS) lock = Lock() logger_cache[filename] = (logger, lock) # N.B. EVERY record gets logged. This is because Logger.handle # is normally called AFTER logger-level filtering. If you want # to do filtering, do it at the client end to save wasting # cycles and network bandwidth! with lock: logger.handle(record)
def handle(self): """ Handle multiple requests - each expected to be a 4-byte length, followed by the LogRecord in pickle format. Logs the record according to whatever policy is configured locally. """ global logger_cache while True: chunk = self.connection.recv(4) if len(chunk) < 4: break slen = struct.unpack(">L", chunk)[0] chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + self.connection.recv(slen - len(chunk)) obj = pickle.loads(chunk) record = logging.makeLogRecord(obj) # if a name is specified, we use the named logger rather than the one # implied by the record. if self.server.logname is not None: name = self.server.logname else: name = record.name if not name: filename = logs_path.join("sprout.log") else: fields = name.split(".") fields[-1] += ".log" filename = logs_path for field in fields: filename = filename.join(field) if not field.endswith(".log"): with global_fs_lock: if not filename.exists(): filename.mkdir() filename = filename.strpath with logger_cache_lock: if filename in logger_cache: logger, lock = logger_cache[filename] else: logger = create_logger( name, filename=filename, max_file_size=MAX_FILE_SIZE, max_backups=MAX_BACKUPS ) lock = Lock() logger_cache[filename] = (logger, lock) # N.B. EVERY record gets logged. This is because Logger.handle # is normally called AFTER logger-level filtering. If you want # to do filtering, do it at the client end to save wasting # cycles and network bandwidth! with lock: logger.handle(record)
def __init__(self, game_cls, rewards=None): """Create an environnemnt given a game class. Args: game_cls (cls of games.game.Game) rewards (dict): @see Environment.rewards """ self.log = create_logger(self.__class__.__name__, LOG_LEVEL) self.rewards = dict(Environment.rewards) if rewards: self.rewards.update(rewards) self.log.info( "Playing %s with %s." % (game_cls.__name__, self.rewards) ) self.game_cls = game_cls self.game = game_cls()
def start_test(self, artifact_path, test_name, test_location, slaveid): if not slaveid: slaveid = "Master" test_ident = "{}/{}".format(test_location, test_name) if slaveid in self.tests: if self.tests[slaveid].in_progress: print "Test already running, can't start another, logger" return None self.tests[slaveid] = self.Test(test_ident) self.tests[slaveid].in_progress = True artifacts = [] os_filename = self.ident + "-" + "cfme.log" os_filename = os.path.join(artifact_path, os_filename) if os.path.isfile(os_filename): os.remove(os_filename) artifacts.append(os_filename) self.tests[slaveid].logger = create_logger(self.ident + test_name, os_filename) self.tests[slaveid].logger.setLevel(self.level) return None, {'artifacts': {test_ident: {'files': {self.ident: artifacts}}}}
from utils.conf import docker as docker_conf from utils.appliance import Appliance from utils.trackerbot import api from utils.log import create_logger from slumber.exceptions import HttpClientError token = docker_conf['gh_token'] owner = docker_conf['gh_owner'] repo = docker_conf['gh_repo'] tapi = api() CONT_LIMIT = docker_conf['workers'] DEBUG = docker_conf.get('debug', False) logger = create_logger('check_prs', 'prt.log') def perform_request(url): """ Simple function to assist in performing GET requests from github Runs if there is a token, else just spits out a blank dict Args: url: The url to process, which is anything after the "...repos/" """ out = {} if token: headers = {'Authorization': 'token {}'.format(token)} full_url = "https://api.github.com/repos/{}/{}/{}".format(owner, repo, url)
about the collector. """ import pytest import signal import subprocess import time from cfme.configure import configuration from utils.conf import env from utils.log import create_logger from utils.net import random_port, my_ip_address, net_check_remote from utils.path import scripts_path from utils.smtp_collector_client import SMTPCollectorClient logger = create_logger('emails') @pytest.yield_fixture(scope="session") def _smtp_test_session(request): """Fixture, which prepares the appliance for e-mail capturing tests Returns: :py:class:`util.smtp_collector_client.SMTPCollectorClient` instance. """ logger.info("Preparing start for e-mail collector") ports = env.get("mail_collector", {}).get("ports", {}) mail_server_port = ports.get("smtp", None) or random_port() mail_query_port = ports.get("json", None) or random_port() my_ip = my_ip_address() logger.info("Mind that it needs ports %d and %d open" % (mail_query_port, mail_server_port)) smtp_conf = configuration.SMTPSettings(
def log(self): return create_logger("rest")
""" import pytest import signal import subprocess import time from cfme.configure import configuration from fixtures.artifactor_plugin import art_client, get_test_idents from utils.conf import env from utils.log import create_logger from utils.net import random_port, my_ip_address, net_check_remote from utils.path import scripts_path from utils.smtp_collector_client import SMTPCollectorClient logger = create_logger("emails") @pytest.fixture(scope="function") def smtp_test(request): """Fixture, which prepares the appliance for e-mail capturing tests Returns: :py:class:`util.smtp_collector_client.SMTPCollectorClient` instance. """ logger.info("Preparing start for e-mail collector") ports = env.get("mail_collector", {}).get("ports", {}) mail_server_port = ports.get("smtp", None) or random_port() mail_query_port = ports.get("json", None) or random_port() my_ip = my_ip_address() logger.info("Mind that it needs ports %s and %s open", mail_query_port, mail_server_port) smtp_conf = configuration.SMTPSettings(host=my_ip, port=mail_server_port, auth="none")
def main(): print "Running . . ." # initialize seeds/patterns list seeds = [] snowball_patterns = [] seed_dict = io.parse_seed_file(config.SNOWBALL_SEEDS_FILE) tuples = [] # output tuples/patterns files + main logger tuples_f = open(config.SNOWBALL_TUPLES_FILE, 'a') patterns_f = open(config.SNOWBALL_PATTERNS_FILE, 'a') logger = log.create_logger("snowball", "snowball.log") # partition sentence search space by iteration sentence_count = config.SNOWBALL_SENTENCE_CAP sentences_per_iter = sentence_count / config.SNOWBALL_NUM_ITERATIONS remainder = sentence_count % config.SNOWBALL_NUM_ITERATIONS counts = [sentences_per_iter] * config.SNOWBALL_NUM_ITERATIONS counts[-1] += remainder # tack on remainder # create initial seed and tuple sets for (subj, obj) in seed_dict['pairs']: tup = classes.CandidateTuple(seed_dict['rel'], subj, obj, seed_dict['subj_tag'], seed_dict['obj_tag'], 1.0) seeds.append(tup) tuples.append(tup) # some info on seeds and relation settings logger.info("Relation: %r, Subject tag: %r, Object Tag: %r", seed_dict['rel'], seed_dict['subj_tag'], seed_dict['obj_tag']) logger.info("Seeds: %r", seed_dict['pairs']) # begin for i in xrange(config.SNOWBALL_NUM_ITERATIONS): logger.info("Beginning iteration %r", i+1) raw_patterns = [] candidate_tuples = {tup: {'matches': [], 'raw_patterns': []} for tup in seeds} logger.info("PATTERN EXTRACTION PHASE") # retrieve sentences / extract raw patterns logger.info("Retrieving sentences and extracting raw patterns . . .") for tup in seeds: count = es.count_sentences_containing(tup) hits = es.get_sentences_containing(tup, 0, count) for hit in hits: source = hit['_source'] sent = classes.Sentence(source['id'], source['index'], source['tokens'], source['tagged_tokens']) raw_patterns.extend(sent.extract_raw_patterns(tup)) logger.info("Number of raw patterns: %d", len(raw_patterns)) # cluster raw patterns logger.info("Clustering raw patterns . . .") clusterer = classes.SinglePassClusteringAlgorithm(raw_patterns, config.SNOWBALL_MIN_PATTERN_SIMILARITY) clusterer.prepare() clusterer.cluster() new_snowball_patterns = clusterer.get_snowball_patterns() if len(new_snowball_patterns) == 0: logger.info("No new patterns. Prematurely exiting on iteration %d", i+1) break snowball_patterns.extend(new_snowball_patterns) logger.info("Total number of patterns: %d", len(snowball_patterns)) logger.info("Writing new patterns to: %r", config.SNOWBALL_TUPLES_FILE) for pat in new_snowball_patterns: io.write_line(patterns_f, pat) logger.info("TUPLE EXTRACTION PHASE") logger.info("Searching through sentences . . .") # retrieve sentences one at a time to avoid blowing up memory for j in xrange(counts[i]): from_offset = j + i*sentences_per_iter source = es.get_sentences_with_tags(seed_dict['subj_tag'], seed_dict['obj_tag'], from_offset, 1)[0]['_source'] sent = classes.Sentence(source['id'], source['index'], source['tokens'], source['tagged_tokens']) candidates = sent.extract_candidate_tuples(seed_dict['rel'], seed_dict['subj_tag'], seed_dict['obj_tag']) for (candidate, raw_pattern) in candidates: best_similarity = 0.0 best_pattern = None for sb_pattern in snowball_patterns: similarity = sb_pattern.match(raw_pattern) if similarity >= config.SNOWBALL_MIN_PATTERN_SIMILARITY: sb_pattern.update_confidence(candidate, seeds) if similarity >= best_similarity: best_similarity = similarity best_pattern = sb_pattern if best_similarity >= config.SNOWBALL_MIN_PATTERN_SIMILARITY: if candidate not in candidate_tuples: candidate_tuples[candidate] = {'matches': [], 'raw_patterns': []} candidate_tuples[candidate]['matches'].append((best_similarity, best_pattern)) candidate_tuples[candidate]['raw_patterns'].append(raw_pattern) new_tuples = candidate_tuples.keys() logger.info("Number of candidate tuples: %d", len(new_tuples)) for new_tuple in new_tuples: new_tuple.update_confidence(candidate_tuples[new_tuple]['matches'], snowball_patterns) new_tuple.add_patterns(candidate_tuples[new_tuple]['raw_patterns']) seeds = [tup for tup in new_tuples if tup.confidence() >= config.SNOWBALL_MIN_TUPLE_CONFIDENCE] if len(seeds) == 0: logger.info("No new seeds. Prematurely exiting on iteration %d", i+1) break tuples.extend(seeds) # store everything logger.info("Number of seed tuples: %d", len(seeds)) logger.info("Total tuples so far (roughly): %d", len(tuples)) logger.info("Ending iteration %d", i+1) logger.info("Writing tuples to: %r", config.SNOWBALL_TUPLES_FILE) for tup in tuples: io.write_line(tuples_f, tup) # close files tuples_f.close() patterns_f.close() print "Done!"
import pandas as pd from sklearn.model_selection import StratifiedKFold, train_test_split from sklearn.utils import shuffle from sklearn.utils.class_weight import compute_class_weight from tabulate import tabulate import torch from torch import nn from activity_data_labeler import (LABEL_DAILY, LABEL_ITEMS, LABEL_ITEMS_INDEX_DICT, LABEL_OTHER_SPORTS) from utils import log, plotting from utils.common import ewma, load_dataset from utils.model_utils import GeneralModelPredictor from utils.trainer import data_normalize, normalize_data, training logger = log.create_logger('ActivityResearch', level=logging.DEBUG) # Other type from type name to int label LABEL_OTHER = LABEL_DAILY + LABEL_OTHER_SPORTS LABEL_BRISKING = ['BriskWalkInDoor', 'BriskWalkOutSide', 'SlowWalk'] LABEL_RUNNING = ['RuningInDoor', 'RuningOutSide'] LABEL_BIKING = ['BikingOutSide'] LABEL_ROWING = ['RowingMachine'] LABEL_ELLIPTICAL = ['EllipticalMachine'] LABEL_IN_USE = LABEL_OTHER + LABEL_BRISKING + LABEL_RUNNING + LABEL_BIKING + LABEL_ROWING + LABEL_ELLIPTICAL LABEL_IN_USE_INDEX = [ LABEL_ITEMS_INDEX_DICT.get(name) for name in LABEL_IN_USE ] # Map the origin index to activity type for model training
def logger(): return create_logger("sprout")
def main(args): with open("./configs/{}/{}_{}_{}.yaml".format(args.net, args.dataset, args.backbone, args.mode)) as fp: cfg = yaml.safe_load(fp) NET_ARGS = cfg['NET_ARGS'] DATA_ARGS = cfg['DATA_ARGS'] EXP_ARGS = cfg['EXP_ARGS'] os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' model_dir = os.path.join('./saved_models/', args.dataset, args.backbone, args.net, args.mode) makedir(model_dir) log, logclose = create_logger(log_filename=os.path.join( model_dir, 'train_logger_{}.txt'.format( datetime.datetime.now().strftime("%H:%M:%S")))) img_dir = os.path.join(model_dir, 'img') makedir(img_dir) weight_matrix_filename = 'outputL_weights' prototype_img_filename_prefix = 'prototype-img' prototype_self_act_filename_prefix = 'prototype-self-act' proto_bound_boxes_filename_prefix = 'bb' log(pformat(cfg)) # ---------------------------------------- Get DataLoaders ---------------------------------------------- normalize = transforms.Normalize(mean=NET_ARGS['mean'], std=NET_ARGS['std']) train_transforms = transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), normalize, ]) train_dataset = datasets.ImageFolder(DATA_ARGS['train_dir'], train_transforms) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=EXP_ARGS['train_batch_size'], shuffle=True, num_workers=4, pin_memory=False) train_push_dataset = datasets.ImageFolder( DATA_ARGS['train_push_dir'], transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), ])) train_push_loader = torch.utils.data.DataLoader( train_push_dataset, batch_size=EXP_ARGS['train_push_batch_size'], shuffle=False, num_workers=4, pin_memory=False) test_dataset = datasets.ImageFolder( DATA_ARGS['test_dir'], transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), normalize, ])) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=EXP_ARGS['test_batch_size'], shuffle=False, num_workers=4, pin_memory=False) log('training set size: {0}'.format(len(train_loader.dataset))) log('push set size: {0}'.format(len(train_push_loader.dataset))) log('test set size: {0}'.format(len(test_loader.dataset))) log('batch size: {0}'.format(EXP_ARGS['train_batch_size'])) # ------------------------------------ Model and Optimizer ---------------------------------------------- ppnet = model_AttProto.construct_PPNet( base_architecture=NET_ARGS['base_architecture'], pretrained=True, img_size=DATA_ARGS['img_size'], prototype_shape=NET_ARGS['prototype_shape'], num_classes=DATA_ARGS['num_classes'], prototype_activation_function=NET_ARGS[ 'prototype_activation_function'], add_on_layers_type=NET_ARGS['add_on_layers_type'], att_version=NET_ARGS['ATT_VERSION']) ppnet = ppnet.cuda() ppnet_multi = torch.nn.DataParallel(ppnet) class_specific = True if EXP_ARGS['RESUME']['iS_RESUME']: ppnet = torch.load(EXP_ARGS['RESUME']['PATH']) log(" Resumed from model: {}".format(EXP_ARGS['RESUME']['PATH'])) ppnet_multi = torch.nn.DataParallel(ppnet) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=True, log=log, EXP_ARGS=EXP_ARGS) log("\nInit Accuracy {:.2f} \n\n".format(accu)) ppnet_multi = torch.nn.DataParallel(ppnet) warm_optimizer_lrs = EXP_ARGS['OPTIMIZER']['warm_optimizer_lrs'] warm_optimizer_specs = [ { 'params': ppnet.add_on_layers.parameters(), 'lr': warm_optimizer_lrs['add_on_layers'], 'weight_decay': 1e-3 }, { 'params': ppnet.prototype_vectors, 'lr': warm_optimizer_lrs['prototype_vectors'] }, { 'params': ppnet.att_layer.parameters(), 'lr': warm_optimizer_lrs['att_layer'], 'weight_decay': 1e-3 }, ] warm_optimizer = torch.optim.Adam(warm_optimizer_specs) joint_optimizer_lrs = EXP_ARGS['OPTIMIZER']['joint_optimizer_lrs'] joint_optimizer_specs = [{ 'params': ppnet.features.parameters(), 'lr': joint_optimizer_lrs['features'], 'weight_decay': 1e-3 }, { 'params': ppnet.add_on_layers.parameters(), 'lr': joint_optimizer_lrs['add_on_layers'], 'weight_decay': 1e-3 }, { 'params': ppnet.prototype_vectors, 'lr': joint_optimizer_lrs['prototype_vectors'] }, { 'params': ppnet.att_layer.parameters(), 'lr': joint_optimizer_lrs['att_layer'], 'weight_decay': 1e-3 }] joint_optimizer = torch.optim.Adam(joint_optimizer_specs) joint_lr_scheduler = torch.optim.lr_scheduler.StepLR( joint_optimizer, step_size=int(joint_optimizer_lrs['joint_lr_step_size']), gamma=0.1) push_epochs = [ i for i in range(EXP_ARGS['num_train_epochs']) if i % 10 == 0 ] log('\n\n------------------------ Start Training ----------------------------\n\n' ) max_acc = 0.0 max_acc_epoch = 0 max_acc_iter = 0 target_accu = 0.1 for epoch in range(EXP_ARGS['start_epoch'], EXP_ARGS['num_train_epochs']): log('------------------------- Epoch: {} -------------------------------------' .format(epoch)) if epoch < EXP_ARGS['num_warm_epochs']: tnt.warm_only(model=ppnet_multi, log=log) _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=warm_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_warm'], log=log, EXP_ARGS=EXP_ARGS) else: tnt.joint(model=ppnet_multi, log=log) joint_lr_scheduler.step() _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=joint_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_joint'], log=log, EXP_ARGS=EXP_ARGS) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) if accu > max_acc: max_acc = accu max_acc_iter = 0 max_acc_epoch = epoch save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='prepush_{}'.format(epoch)) log("\nBest Accuracy {:.2f} at epoch {} and iter {}\n\n".format( max_acc, max_acc_epoch, max_acc_iter)) if epoch >= EXP_ARGS['push_start'] and epoch in push_epochs: save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='prepushfinal_{}'.format(epoch)) log('\n------------------------- Push Prototypes -----------------------------' ) push.push_prototypes( train_push_loader, prototype_network_parallel=ppnet_multi, class_specific=class_specific, preprocess_input_function=preprocess_input_function, prototype_layer_stride=1, root_dir_for_saving_prototypes=img_dir, epoch_number=epoch, prototype_img_filename_prefix=prototype_img_filename_prefix, prototype_self_act_filename_prefix= prototype_self_act_filename_prefix, proto_bound_boxes_filename_prefix= proto_bound_boxes_filename_prefix, save_prototype_class_identity=True, log=log) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='push_{}'.format(epoch)) last_layer_optimizer_specs = [{ 'params': ppnet.last_layer.parameters(), 'lr': EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_layer_optimizer_lr'] }] last_layer_optimizer = torch.optim.Adam(last_layer_optimizer_specs) last_lr_lr_scheduler = torch.optim.lr_scheduler.StepLR( last_layer_optimizer, step_size=EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_lr_step_size'], gamma=0.1) log('\n------------------------- Last Layer Training -----------------------------------' ) if NET_ARGS['prototype_activation_function'] != 'linear': tnt.last_only(model=ppnet_multi, log=log) max_acc_post, max_acc_post_iter, max_acc_post_epoch = 0, 0, epoch for i in range( EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_layer_optimizer_iters']): log('Last layer optimization, Iteration: {0}'.format(i)) _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=last_layer_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_joint'], log=log, EXP_ARGS=EXP_ARGS) last_lr_lr_scheduler.step() accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) if accu > max_acc_post: max_acc_post = accu max_acc_post_iter = i max_acc_post_epoch = epoch save.save_model_w_condition( model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=0.70, log=log, best=True, stage='postpush_{}'.format(epoch)) log("Best Accuracy - PostPush {:.2f} at epoch {} and iter {}" .format(max_acc_post, max_acc_post_epoch, max_acc_post_iter)) save.save_model_w_condition( model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=0.70, log=log, best=True, stage='postpushfinal_{}'.format(epoch)) logclose()
parser = argparse.ArgumentParser(description='xxx') parser.add_argument('--config', type=str, default='cfg/train_birds.yml') args = parser.parse_args() cfg_from_file(args.config) os.environ['CUDA_VISIBLE_DEVICES'] = cfg.GPU_ID # Set directories and logger now = datetime.datetime.now(dateutil.tz.tzlocal()) timestamp = now.strftime('%Y_%m_%d_%H_%M_%S') output_dir = 'sample/%s_%s_%s' % (cfg.DATASET_NAME, cfg.CONFIG_NAME, timestamp) log_dir = os.path.join( cfg.LOG_DIR, '%s_%s_%s' % (cfg.DATASET_NAME, cfg.CONFIG_NAME, timestamp)) os.makedirs(log_dir, exist_ok=True) log_filename = os.path.join(log_dir, 'train.log') log, logclose = create_logger(log_filename=log_filename) writer = SummaryWriter(log_dir=log_dir) with open(log_filename, 'w+') as logFile: pprint.pprint(cfg, logFile) log('') log('============================================================================' ) log('') log('Dataset:') image_transform = transforms.Compose([transforms.Resize((128, 128))]) train_dataset = CUBDataset(cfg.DATA_DIR, transform=image_transform, split='train')
import config as cfg
def logger(): return create_logger("sprout_vm_actions")
def main(): global args, best_EPE, save_path args = parser.parse_args() # Load config file if args.cfg is not None: cfg_from_file(args.cfg) assert cfg.TAG == os.path.splitext(os.path.basename( args.cfg))[0], 'TAG name should be file name' # Build save_path, which can be specified by out_dir and exp_dir save_path = '{},{}epochs{},b{},lr{}'.format( 'dicl_wrapper', args.epochs, ',epochSize' + str(args.epoch_size) if args.epoch_size > 0 else '', args.batch_size, args.lr) save_path = os.path.join(args.exp_dir, save_path) if args.out_dir is not None: outpath = os.path.join(args.out_dir, args.dataset) else: outpath = args.dataset save_path = os.path.join(outpath, save_path) if not os.path.exists(outpath): os.makedirs(outpath) if not os.path.exists(save_path): os.makedirs(save_path) # Create logger log_file = os.path.join(save_path, 'log.txt') logger = create_logger(log_file) logger.info('**********************Start logging**********************') logger.info('=> will save everything to {}'.format(save_path)) # Print settings for _, key in enumerate(args.__dict__): logger.info(args.__dict__[key]) save_config_to_file(cfg, logger=logger) logger.info(args.pretrained) # Set random seed torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) train_writer = SummaryWriter(os.path.join(save_path, 'train')) eval_writer = SummaryWriter(os.path.join(save_path, 'eval')) logger.info("=> fetching img pairs in '{}'".format(args.data)) ########################## DATALOADER ########################## if args.dataset == 'flying_chairs': if cfg.SIMPLE_AUG: train_dataset = datasets.FlyingChairs_SimpleAug(args, root=args.data) test_dataset = datasets.FlyingChairs_SimpleAug(args, root=args.data, mode='val') else: train_dataset = datasets.FlyingChairs(args, image_size=cfg.CROP_SIZE, root=args.data) test_dataset = datasets.FlyingChairs(args, root=args.data, mode='val', do_augument=False) elif args.dataset == 'flying_things': train_dataset = datasets.SceneFlow(args, image_size=cfg.CROP_SIZE, root=args.data, dstype='frames_cleanpass', mode='train') test_dataset = datasets.SceneFlow(args, image_size=cfg.CROP_SIZE, root=args.data, dstype='frames_cleanpass', mode='val', do_augument=False) elif args.dataset == 'mpi_sintel_clean' or args.dataset == 'mpi_sintel_final': clean_dataset = datasets.MpiSintel(args, image_size=cfg.CROP_SIZE, root=args.data, dstype='clean') final_dataset = datasets.MpiSintel(args, image_size=cfg.CROP_SIZE, root=args.data, dstype='final') train_dataset = torch.utils.data.ConcatDataset([clean_dataset] + [final_dataset]) if args.dataset == 'mpi_sintel_final': test_dataset = datasets.MpiSintel(args, do_augument=False, image_size=None, root=args.data, dstype='final') else: test_dataset = datasets.MpiSintel(args, do_augument=False, image_size=None, root=args.data, dstype='clean') elif args.dataset == 'KITTI': train_dataset = datasets.KITTI(args, image_size=cfg.CROP_SIZE, root=args.data, is_val=False, logger=logger) if args.data_kitti12 is not None: train_dataset12 = datasets.KITTI12(args, image_size=cfg.CROP_SIZE, root=args.data_kitti12, is_val=False, logger=logger) train_dataset = torch.utils.data.ConcatDataset([train_dataset] + [train_dataset12]) test_dataset = datasets.KITTI(args, root=args.data, do_augument=False, is_val=True, do_pad=False) else: raise NotImplementedError logger.info('Training with %d image pairs' % len(train_dataset)) logger.info('Testing with %d image pairs' % len(test_dataset)) gpuargs = {'num_workers': args.workers, 'drop_last': cfg.DROP_LAST} train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, pin_memory=True, shuffle=True, **gpuargs) if 'KITTI' in args.dataset: # We set batch size to 1 since KITTI images have different sizes val_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, num_workers=args.workers, pin_memory=True, shuffle=False) else: val_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True, shuffle=False) # create model if args.pretrained: logger.info("=> using pre-trained model '{}'".format(args.pretrained)) pretrained_dict = torch.load(args.pretrained) if 'state_dict' in pretrained_dict.keys(): pretrained_dict['state_dict'] = { k: v for k, v in pretrained_dict['state_dict'].items() } model = models.__dict__['dicl_wrapper'](None) assert (args.solver in ['adam', 'sgd']) logger.info('=> setting {} solver'.format(args.solver)) if args.solver == 'adam': optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=cfg.WEIGHT_DECAY, betas=(cfg.MOMENTUM, cfg.BETA)) elif args.solver == 'sgd': optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, weight_decay=cfg.WEIGHT_DECAY, momentum=cfg.MOMENTUM) if args.pretrained: if 'state_dict' in pretrained_dict.keys(): model.load_state_dict(pretrained_dict['state_dict'], strict=False) else: model.load_state_dict(pretrained_dict, strict=False) if args.reuse_optim: try: optimizer.load_state_dict(pretrained_dict['optimizer_state']) except: logger.info('do not have optimizer state') del pretrained_dict torch.cuda.empty_cache() model = torch.nn.DataParallel(model) if torch.cuda.is_available(): model = model.cuda() # Evaluation if args.evaluate: with torch.no_grad(): best_EPE = validate(val_loader, model, 0, None, eval_writer, logger=logger) return # Learning rate schedule milestones = [] for num in range(len(args.milestones)): milestones.append(int(args.milestones[num])) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.5) ###################################### Training ###################################### for epoch in range(args.start_epoch, args.epochs): # train for one epoch train_loss = train(train_loader, model, optimizer, epoch, train_writer, logger=logger) scheduler.step() train_writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) train_writer.add_scalar('avg_loss', train_loss, epoch) if epoch % args.eval_freq == 0 and not args.no_eval: with torch.no_grad(): EPE = validate(val_loader, model, epoch, output_writers, eval_writer, logger=logger) eval_writer.add_scalar('mean_EPE', EPE, epoch) if best_EPE < 0: best_EPE = EPE if EPE < best_EPE: best_EPE = EPE ckpt_best_file = 'checkpoint_best.pth.tar' save_checkpoint( { 'epoch': epoch + 1, 'arch': 'dicl_wrapper', 'state_dict': model.module.state_dict(), 'optimizer_state': optimizer.state_dict(), 'best_EPE': EPE }, False, filename=ckpt_best_file) logger.info('Epoch: [{0}] Best EPE: {1}'.format(epoch, best_EPE)) # Skip at least 5 epochs to save memory save_freq = max(args.eval_freq, 5) if epoch % save_freq == 0: ckpt_file = 'checkpoint_' + str(epoch) + '.pth.tar' save_checkpoint( { 'epoch': epoch + 1, 'arch': 'dicl_wrapper', 'state_dict': model.module.state_dict(), 'optimizer_state': optimizer.state_dict(), 'best_EPE': best_EPE }, False, filename=ckpt_file)
from utils.appliance import Appliance from utils.trackerbot import api from utils.log import create_logger from slumber.exceptions import HttpClientError token = docker_conf["gh_token"] owner = docker_conf["gh_owner"] repo = docker_conf["gh_repo"] tapi = api() CONT_LIMIT = docker_conf["workers"] DEBUG = docker_conf.get("debug", False) logger = create_logger("check_prs", "prt.log") # Disable pika logs logging.getLogger("pika").propagate = False def send_message_to_bot(msg): required_fields = set(["rabbitmq_url", "gh_queue", "gh_channel", "gh_message_type"]) if not required_fields.issubset(docker_conf.viewkeys()): logger.warn("Skipping - docker.yaml doesn't have {}".format(required_fields)) return logger.info("Github PR bot: about to send '{}'".format(msg)) url = docker_conf["rabbitmq_url"] queue = docker_conf["gh_queue"]
about the collector. """ import pytest import signal import subprocess import time from cfme.configure import configuration from fixtures.artifactor_plugin import art_client, get_test_idents from utils.conf import env from utils.log import create_logger from utils.net import random_port, my_ip_address, net_check_remote from utils.path import scripts_path from utils.smtp_collector_client import SMTPCollectorClient logger = create_logger('emails') @pytest.fixture(scope="function") def smtp_test(request): """Fixture, which prepares the appliance for e-mail capturing tests Returns: :py:class:`util.smtp_collector_client.SMTPCollectorClient` instance. """ logger.info("Preparing start for e-mail collector") ports = env.get("mail_collector", {}).get("ports", {}) mail_server_port = ports.get("smtp", None) or random_port() mail_query_port = ports.get("json", None) or random_port() my_ip = my_ip_address() logger.info("Mind that it needs ports {} and {} open".format( mail_query_port, mail_server_port))
from enum import Enum, unique import json import os from pathlib import Path import PySimpleGUI as sg import click import matplotlib.pyplot as plt import numpy as np import pandas as pd from utils import log from utils.data_labeler import DataLabeler, merge_labels logger = log.create_logger('ActiDataLabeler') SENSOR_DATA_NAMES = ['type', 'ts', 'x', 'y', 'z', 'xe', 'ye', 'ze'] SENSOR_DATA_HEADER = [ 'CurrentTimeMillis', 'EventTimestamp(ns)', 'accel_x', 'accel_y', 'accel_z' ] CTM_HEADER_NAME = 'CurrentTimeMillis' TS_HEADER_NAME = 'EventTimestamp(ns)' TSS = [CTM_HEADER_NAME, TS_HEADER_NAME] # !! In order to be compatible with historical annotations, please add the type at the end instead of inserting # Refer to list: https://xiaomi.f.mioffice.cn/docs/dock4H2cyIYjwEGAnMmFH5GOcTc# LABEL_ITEMS = ( 'Undefined', # 0 'Static', # 1 'DailyActivity', # 2, Such as Working/Writing/Chatting
def __init__(self, i, j): self.content = None self.position = Position(i, j) self.log = create_logger(self.__class__.__name__, log_level=LOG_LEVEL)
from cached_property import cached_property from fixtures.artifactor_plugin import art_client, get_test_idents from fixtures.terminalreporter import reporter from utils import providers from utils.conf import env from utils.db import cfmedb from utils.datafile import template_env from utils.events import setup_for_event_testing from utils.log import create_logger from utils.net import my_ip_address, random_port from utils.path import scripts_path from utils.ssh import SSHClient from utils.wait import wait_for, TimedOutError logger = create_logger('events') def get_current_time_GMT(): """ Because SQLite loves GMT. Returns: datetime() with current GMT time """ return datetime.strptime(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), "%Y-%m-%d %H:%M:%S") class HTMLReport(object): def __init__(self, test_name, registered_events, all_events): self.registered_events = registered_events self.test_name = test_name
def api_logger(): return create_logger("sprout_api")