import os
from loguru import logger
from .settings import BASE_DIR
logger.remove(handler_id=None)
log_file = os.path.join(BASE_DIR, "log/file_{time}.log")
logger.add(log_file, backtrace=True, retention="5 min", encoding="utf8")
from lib.config import DEBUG_LOG
from lib.resize import max_resize
from widgets.main_window import main_window
if __name__ == "__main__":
args = sys.argv
# start app
appctxt = ApplicationContext()
app = appctxt.app
# prepare the logger
logger.add(
DEBUG_LOG,
rotation="1 MB",
retention="1 week",
backtrace=True,
diagnose=True,
enqueue=True,
)
logger.info("-----------------------")
logger.info("Launching application")
try:
# create instance of main window
main_window = main_window(app, appctxt)
# build the main window
main_window.build()
main_window.set_theme()
# load data
main_window.main_widget.find_sim()
def main(arguments: argparse.Namespace) -> ExitCode:
logger.remove()
logger.add(sys.stderr, level=arguments.logging_level)
logger.debug("Called with {!r}", arguments)
if not (arguments.postgres_connection_string
and arguments.synapse_auth_token):
logger.error(
"Postgres connection string and Synapse auth token can not be empty"
)
return ExitCode.Failure
try:
db = postgres.Postgres(arguments.postgres_connection_string)
except (psycopg2.OperationalError, psycopg2.InternalError):
logger.exception(
"Connecting to database using {!r} failed:",
arguments.postgres_connection_string,
)
return ExitCode.Failure
session = requests.session()
session.headers["Authorization"] = "Bearer " + arguments.synapse_auth_token
before_date = get_delta_date(int(arguments.delta))
before_date_string = before_date.isoformat(sep=" ", timespec="seconds")
logger.info("Purging events up to: {} UTC", before_date_string)
# Purge room history for all rooms
rooms = get_room_record_ids(db)
room_count = len(rooms)
for index, room_id in enumerate(rooms, start=1):
logger.info("({}/{}) Processing room: {!r}...", index, room_count,
room_id)
event_id = get_last_event_id(db, room_id, before_date)
if event_id is None:
logger.info(
"No event ID before: {} UTC for room: {!r}, skipping",
before_date_string,
room_id,
)
continue
logger.info(
"Last event ID before: {} UTC for room {!r}: {!r}",
before_date_string,
room_id,
event_id,
)
purge_id = purge_history(session, arguments.api_url, room_id, event_id)
logger.info("Purging room: {!r} in progress: {!r}...", room_id,
purge_id)
result = wait_for_purge(session, arguments.api_url, purge_id)
logger.info("Purged room: {!r} with status: {!r}", room_id, result)
logger.info("Purging local media older than: {} UTC...",
before_date_string)
local_media = get_local_media_record_ids(db, before_date)
important_files = get_important_media_ids(db)
# Purge local media manually
old_media = set(local_media) - important_files
old_media_count = len(old_media)
logger.info("{} media to be cleaned...", len(old_media))
for index, media_id in enumerate(old_media, start=1):
logger.info("({}/{}) Processing media: {!r}...", index,
old_media_count, media_id)
paths = get_local_media_paths(arguments.media_store, media_id)
for path in paths:
if not os.path.isfile(path):
logger.debug("{!r} could not be found or is not a file", path)
continue
os.remove(path)
delete_local_media_record(db, media_id)
# Purge remote media
logger.info("Purging cached remote media older than: {} UTC...",
before_date_string)
result = purge_remote_media(session, arguments.api_url, before_date)
logger.info("Purged cached remote media: {!r}", result)
return ExitCode.Success
for _ in range(self.CONCURREND_DOWNLOADS):
token_queue.put(token)
logger.debug("ETLWorker | read Token and prepared Queues")
return token_queue, insert_queue
def load(self, id: int, token_queue: mp.Queue):
logger.debug("Get token")
token = token_queue.get()
logger.info(f"Load data with pseudo_token {self.token}")
response = requests.get(f"https://pokeapi.co/api/v2/pokemon/{id}")
logger.info(f"Received answer with code {response.status_code}")
token_queue.put(token)
logger.debug("Put token")
self.data = response.json()
def transform(self) -> Pokemon:
logger.debug("Start Transformation")
time.sleep(10) # make this function a little more expansive
id = self.data["id"]
name = self.data["name"]
base_stat = self.data["stats"][0]["base_stat"]
return Pokemon(id=id, name=name, base_stat=base_stat)
if __name__ == '__main__':
logger.level("DEBUG")
logger.add("download.log")
etl = ETLWorker()
etl.main()
logger.success("Done")
# -*- coding: utf-8 -*-
"""
-------------------------------------------------
File Name： test_log
Description :
Author : xmz
date： 2019/4/22
-------------------------------------------------
"""
import os
import sys
from loguru import logger
logger.remove()
logger.add(sys.stdout, colorize=True,
format="<yellow>{level}</yellow> | <red>{file}</red> | <red>{module}</red> | " \
"<green>{time:YYYY-MM-DD at HH:mm:ss}</green> | <level>{message}</level>",
level="INFO")
logger.add(
os.path.join("../../logs", "log_{time}.txt"),
enqueue=True,
colorize=True,
format=
"{level} | {file} | {module} | {time:YYYY-MM-DD at HH:mm:ss} | {message}",
level="INFO")
logger.info("sth")
logger.info("sb")
logger.info("If you're using Python {:.2f}, prefer of course!", 3.6333)
from logging import exception
import requests
import pandas as pd
from loguru import logger
logger.add("ygezuzhu.log",encoding="utf8")
class ToExcle():
"""生成exlce方法"""
@classmethod
def dict_to_excle(self, file_path=None, rows=None):
"""
data/sql-->dict-->excle
数据类型为字典转excle
"""
# 1.取数据
df1 = pd.DataFrame(rows)
# 2.保存至excel文件
writer = pd.ExcelWriter(file_path)
df1.to_excel(writer, encoding='utf8', index=False)
# 6.保存
writer.save()
@classmethod
def tuple_to_excle(cls, file_path=None, rows=None, columns=None):
"""
data/sql-->tuple-->excle
数据类型为元组/列表转excle
from nltk.tokenize import sent_tokenize
from pickle import load
from langdetect import detect
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from sklearn.svm import LinearSVC, SVC
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from nltk import ngrams
logger.remove()
logger.add(sys.stdout, level="INFO")
def start_cr_evaluator():
path_to_store_result = os.path.join(os.getcwd(), "cr_evaluator_results")
os.makedirs(path_to_store_result, exist_ok=True)
logger.info(
"Start Analysis to find all privacy policy page without explicit acceptance"
)
# All dir inside log directory
path_log = os.path.join(os.getcwd(), "logs")
dir_inside_log = [x[0] for x in os.walk(path_log)]
dir_inside_log.pop(0)
logger.info(
"Load ML Model for each practices and performed_not_performed and third_party_first_party"
def main():
""" Performance test with locust: parse command line options and run commands.
"""
print(f"HttpRunner version: {__version__}")
sys.argv[0] = 'locust'
if len(sys.argv) == 1:
sys.argv.extend(["-h"])
if sys.argv[1] in ["-h", "--help", "-V", "--version"]:
start_locust_main()
def get_arg_index(*target_args):
for arg in target_args:
if arg not in sys.argv:
continue
return sys.argv.index(arg) + 1
return None
# set logging level
loglevel_index = get_arg_index("-L", "--loglevel")
if loglevel_index and loglevel_index < len(sys.argv):
loglevel = sys.argv[loglevel_index]
loglevel = loglevel.upper()
else:
# default
loglevel = "WARNING"
logger.remove()
logger.add(sys.stdout, level=loglevel)
# get testcase file path
try:
testcase_index = get_arg_index("-f", "--locustfile")
assert testcase_index and testcase_index < len(sys.argv)
except AssertionError:
print("Testcase file is not specified, exit.")
sys.exit(1)
testcase_file_path = sys.argv[testcase_index]
sys.argv[testcase_index] = parse_locustfile(testcase_file_path)
if "--processes" in sys.argv:
""" locusts -f locustfile.py --processes 4
"""
if "--no-web" in sys.argv:
logger.error("conflict parameter args: --processes & --no-web. \nexit.")
sys.exit(1)
processes_index = sys.argv.index('--processes')
processes_count_index = processes_index + 1
if processes_count_index >= len(sys.argv):
""" do not specify processes count explicitly
locusts -f locustfile.py --processes
"""
processes_count = multiprocessing.cpu_count()
logger.warning(f"processes count not specified, use {processes_count} by default.")
else:
try:
""" locusts -f locustfile.py --processes 4 """
processes_count = int(sys.argv[processes_count_index])
sys.argv.pop(processes_count_index)
except ValueError:
""" locusts -f locustfile.py --processes -P 8888 """
processes_count = multiprocessing.cpu_count()
logger.warning(f"processes count not specified, use {processes_count} by default.")
sys.argv.pop(processes_index)
run_locusts_with_processes(sys.argv, processes_count)
else:
start_locust_main()
""" ============================ Author:古一 Time:2020/10/28 E-mail:[email protected] ============================ """ import os import pytest from loguru import logger logger.add('./log/{time}.log', rotation='20 MB', retention='1 week', encoding='utf-8') pytest.main(['-s', r"--alluredir=report/json", "--clean-alluredir"]) os.system('allure generate ./report/json -o ./report/html -c') # pytest.main(['-s'])
def main():
det_data_dir = Path(os.getenv('det_data'))
task_data_dir = det_data_dir / "Task011_Kits"
source_data_dir = task_data_dir / "raw"
if not source_data_dir.is_dir():
raise RuntimeError(
f"{source_data_dir} should contain the raw data but does not exist."
)
splitted_dir = task_data_dir / "raw_splitted"
target_data_dir = task_data_dir / "raw_splitted" / "imagesTr"
target_data_dir.mkdir(exist_ok=True, parents=True)
target_label_dir = task_data_dir / "raw_splitted" / "labelsTr"
target_label_dir.mkdir(exist_ok=True, parents=True)
logger.remove()
logger.add(sys.stdout, level="INFO")
logger.add(task_data_dir / "prepare.log", level="DEBUG")
# save meta info
dataset_info = {
"name": "Kits",
"task": "Task011_Kits",
"target_class": None,
"test_labels": True,
"seg2det_stuff": [
1,
], # define stuff classes: kidney
"seg2det_things": [
2,
], # define things classes: tumor
"min_size": 3.,
"labels": {
"0": "lesion"
},
"labels_stuff": {
"1": "kidney"
},
"modalities": {
"0": "CT"
},
"dim": 3,
}
save_json(dataset_info, task_data_dir / "dataset.json")
# prepare cases
cases = [str(c.name) for c in source_data_dir.iterdir() if c.is_dir()]
for c in maybe_verbose_iterable(cases):
logger.info(f"Copy case {c}")
case_id = int(c.split("_")[-1])
if case_id < 210:
shutil.copy(source_data_dir / c / "imaging.nii.gz",
target_data_dir / f"{c}_0000.nii.gz")
shutil.copy(source_data_dir / c / "segmentation.nii.gz",
target_label_dir / f"{c}.nii.gz")
# create an artificial test split
create_test_split(
splitted_dir=splitted_dir,
num_modalities=1,
test_size=0.3,
random_state=0,
shuffle=True,
)
import json
import requests
from loguru import logger
import time, os
logger.add(
os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs",
"weibo.log"),
level='DEBUG',
format='{time:YYYY-MM-DD HH:mm:ss} - {level} - {file} - {line} - {message}',
rotation="10 MB")
class weiboMonitor(object):
def __init__(self, uuids):
self.uuids = uuids
self.session = requests.session()
self.reqHeaders = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Content-Type': 'application/x-www-form-urlencoded',
'Referer': 'https://passport.weibo.cn/signin/login',
'Connection': 'close',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
}
@logger.catch
def monitor_singer(self, uuid):
user_info = 'https://m.weibo.cn/api/container/getIndex?uid=%s&type=uid&value=%s' % (
uuid, uuid)
con_id = ''
def test_disabled_logger_in_sink(sink_with_logger):
sink = sink_with_logger(logger)
logger.disable("tests.conftest")
logger.add(sink, format="{message}")
logger.info("Disabled test")
assert sink.out == "Disabled test\n"
def test_file_sink_utf8_encoding(tmpdir):
file = tmpdir.join("test.log")
logger.add(str(file), encoding="utf8", format="{message}", errors="strict", catch=False)
logger.info("天")
logger.remove()
assert file.read_text("utf8") == "天\n"
def contextualize_test():
logger.add(get_log_file("context"), format="{extra[ip]} {extra[user]} {message}", serialize=True)
logger_ctx = logger.bind(ip="127.0.0.1", user="******")
logger_ctx.info("Contextualize logger.")
logger_ctx.bind(user="******").info("inline binding other user.")
def test_level_too_high(writer, level):
logger.add(writer, level=level, format="{message}")
logger.info("Test level")
assert writer.read() == ""
import utils
import requests
import json
import os.path
from pprint import pformat
from oauth2creds import get_credentials
from loguru import logger
cfg = utils.config()
logger.add("gphoto_upload.log", rotation="1 MB")
def upload_to_gphotos(filepath, filename=None):
if filename is None:
filename = os.path.basename(filepath)
response = _upload_binary_media(filepath, filename)
if not response.ok:
success = False
elapsed = response.elapsed.microseconds/1_000_000
else:
success, elapsed = _insert_new_photo(response.text)
return success, elapsed
def _upload_binary_media(filepath, filename):
creds = get_credentials()
with open(filepath, "rb") as photo_fp:
binary_file = photo_fp.read()
url = r"https://photoslibrary.googleapis.com/v1/uploads"
headers = {
"Content-type": "application/octet-stream",
def test_invalid_level(writer, level):
with pytest.raises(TypeError):
logger.add(writer, level=level)
from loguru import logger
import sys
logger.remove(0)
logger.add(sys.stdout,
colorize=True,
format="{level}: <level>{message}</level>")
def test_unknown_level(writer, level):
with pytest.raises(ValueError):
logger.add(writer, level=level)
def main():
parser = argparse.ArgumentParser(
description='Scene Classification Training')
parser.add_argument('--device',
default='cuda:0',
type=str,
required=False,
help='GPU ids')
parser.add_argument('--epoch',
default=350,
type=int,
required=True,
help='training epochs')
parser.add_argument('--alldata',
dest='alldata',
action='store_true',
help='use alldata to train')
parser.add_argument('--lr', default=0.01, type=float, help='learning rate')
parser.add_argument('--weight_decay',
'-w',
default=5e-4,
type=float,
help='weight_decay')
parser.add_argument('--batch_size',
default=128,
type=int,
help='training batch size')
parser.add_argument('--output_dir', default='./checkpoint', type=str)
parser.add_argument('--warm_up_epochs', default=10, type=int)
parser.add_argument('--log_file', type=str, default='./log/default.log')
parser.add_argument('--params', type=str, default=None)
args = parser.parse_args()
log_file_name = args.log_file
logger.add(log_file_name)
logger.info('args:\n' + args.__repr__())
batch_size = args.batch_size
output_dir = args.output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
device = args.device
best_acc = 0 # best test accuracy
# Data
logger.info('==> Preparing data..')
transform_train = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.5,
contrast=0.5,
saturation=0.5,
hue=0.5),
transforms.RandomGrayscale(p=0.1),
transforms.ToTensor(),
])
transform_test = transforms.Compose([
transforms.ToPILImage(), #适应自己编写的dataset
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor()
])
train_dir = os.path.join(DATA_PATH, 'train/train')
if args.alldata == False:
whole_set = SceneDataset(
annotations_csv='/home/linhw/myproject/data_scene/train_labels.csv',
root_dir=train_dir,
transform=transform_train)
whole_set2 = SceneDataset(
annotations_csv='/home/linhw/myproject/data_scene/train_labels.csv',
root_dir=train_dir,
transform=transform_test)
whole_len = len(whole_set)
train_len = int(whole_len * 0.9) #划分train和val数据集
val_len = whole_len - train_len
indices = random.sample(range(0, whole_len), train_len)
indices2 = []
for x in range(0, whole_len):
if x not in indices:
indices2.append(x)
trainset = torch.utils.data.Subset(whole_set, indices)
testset = torch.utils.data.Subset(whole_set2, indices2)
else:
trainset = SceneDataset(
annotations_csv='/home/linhw/myproject/data_scene/train_labels.csv',
root_dir=train_dir,
transform=transform_train)
testset = torch.utils.data.Subset(
SceneDataset(annotations_csv=
'/home/linhw/myproject/data_scene/train_labels.csv',
root_dir=train_dir,
transform=transform_test), range(8000)
) # when whole data is True, randomly choose 8000 training samples. Test acc here is training acc.
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=batch_size,
shuffle=True,
num_workers=8)
testloader = torch.utils.data.DataLoader(testset,
batch_size=batch_size,
shuffle=False,
num_workers=8)
# Model
logger.info('==> Building model..')
model = EfficientNet.from_name('efficientnet-b0', num_classes=100)
model = model.to(device)
if args.params is not None:
model.load_state_dict(torch.load(args.params, map_location=device))
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
#optimizer = optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
warm_up_with_cosine_lr = lambda epoch: epoch / args.warm_up_epochs if epoch <= args.warm_up_epochs else 0.5 * (
math.cos((epoch - args.warm_up_epochs) /
(args.epoch - args.warm_up_epochs) * math.pi) + 1)
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lr_lambda=warm_up_with_cosine_lr)
best_acc = 0
for epoch in range(args.epoch):
logger.info("Epoch {} started".format(epoch))
train_acc, training_loss = train(model, optimizer, trainloader, device)
logger.info(
"train acc = {:.4f}, training loss = {:.4f} lr = {:.4f}".format(
train_acc, training_loss, warm_up_with_cosine_lr(epoch)))
test_acc, test_loss = test(model, testloader, device)
logger.info("test acc = {:.4f}, test loss = {:.4f}".format(
test_acc, test_loss))
if test_acc > best_acc:
best_acc = test_acc
logger.info("best acc improved to {:.4f}".format(best_acc))
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(),
'{}/bset_model.pt'.format(output_dir))
logger.info("model saved to {}/bset_model.pt".format(output_dir))
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(),
'{}/last_model.pt'.format(output_dir))
logger.info("model saved to {}/last_model.pt".format(output_dir))
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(),
'{}/model.pt'.format(output_dir))
logger.info("model saved to {}/model.pt".format(output_dir))
scheduler.step()
logger.info("Epoch {} ended, best acc = {:.4f}".format(
epoch, best_acc))
logger.info("Training finished, best_acc = {:.4f}".format(best_acc))
def test_level_low_enough(writer, level):
logger.add(writer, level=level, format="{message}")
logger.info("Test level")
assert writer.read() == "Test level\n"
import sys
from loguru import logger
logger.remove()
logger.add(sys.stderr, format="", colorize=False, backtrace=True)
@logger.catch()
def a():
1 / 0
def b():
2 / 0
def c():
3 / 0
a()
with logger.catch():
b()
try:
c()
except ZeroDivisionError:
logger.exception("")
z_cs = pickle.load(f)
exp_args = load_yaml_args(args.config)
x_cs_discrete = get_discrete_configspace(x_cs, exp_args.x_grid_size)
z_cs_discrete = get_discrete_configspace(z_cs,
exp_args.z_grid_size,
fidelity_space=True)
config_spaces = dict(x=x_cs,
x_discrete=x_cs_discrete,
z=z_cs,
z_discrete=z_cs_discrete)
# Logging details
log_suffix = time.strftime("%x %X %Z")
log_suffix = log_suffix.replace("/", '-').replace(":",
'-').replace(" ", '_')
logger.add("{}/logs/collator_{}.log".format(path, log_suffix),
**_logger_props)
print("Logging at {}/logs/collator_{}.log".format(path, log_suffix))
initial_file_list = os.listdir(path)
# task_datas = dict()
while True:
task_datas = dict()
# list available tasks
task_ids = [
int(tid) for tid in os.listdir(dump_path)
if len(os.listdir(os.path.join(dump_path, tid))) > 0
]
if len(task_ids) == 0:
continue
batch_size = args.max_batch_size // len(task_ids)
from loguru import logger
# 基本使用
logger.debug('this is a debug message')
# 将结果输入到一个runtime。log文件里面
# logger.add('runtime.log')
logger.debug('this is a debug message')
# for x in range(1000):
# logger.info('%s' % x)
# logger.add('runtime1.log', format="{time} {level} {message}", filter="my_module", level="INFO")
# logger.add('runtime1.log', format="{time} {level} {message}", filter="my_module", level="DEBUG")
from loguru import logger
trace = logger.add('runtime2.log')
logger.debug('this is a debug message11')
# logger.remove(trace)
# logger.debug('this is another debug message11')
# rotation 配置
# 通过这样的配置我们就可以实现每 500MB 存储一个文件，每个 log 文件过大就会新创建一个 log 文件。我们在配置 log
# 名字时加上了一个 time 占位符，这样在生成时可以自动将时间替换进去，生成一个文件名包含时间的 log 文件。
# logger.add('runtime_{time}.log', rotation="1024 MB")
# 配置为1k大的文件
# logger.add('runtime_{time}.log', rotation="1024 B")
# logger.add('runtime_{time}.log', rotation="10 MB")
# logger.info()
# for x in range(10000000000000):
# logger.info('%s' % x)
from tqdm import tqdm
from vidaug import augmentors as va
import math
import copy
from sklearn.model_selection import train_test_split, KFold
import functools
from collections import Counter
import torch
import pandas as pd
import torch.utils.data as data
from PIL import Image
DATASET_PATH = '../GTA_dataset'
JPG_PATH = '../GTA_JPG_DATASET'
logger.add(sys.stdout)
def load_dataset_config(config_path)->Dict[str, List[str]]:
with open(config_path) as file:
dataset_conf = yaml.load(file, Loader=yaml.FullLoader)
class_map = dataset_conf['class_map']
return class_map
def pil_loader(path):
# open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
with open(path, 'rb') as f:
with Image.open(f) as img:
return img.convert('RGB')
import backtrader as bt
from math import floor
import logging
from loguru import logger
from initialize import APP_PATH
logging.basicConfig(level=logging.DEBUG,
filename='app.log',
filemode='w',
format='%(name)s - %(levelname)s - %(message)s')
logger.add(
f"{APP_PATH}/logs/update_eri_dates_{datetime.date.today().strftime('%Y%m%d')}.log",
rotation="5 MB",
backtrace=True,
diagnose=True)
class GenericStrategy(bt.Strategy):
params = dict(window_s=21,
window_m=50,
window_l=100,
window_xs=5,
risk=0.05,
stop_dist=0.05,
dev_multiplier=2)
def __init__(self):
# turn on history
def create_logger():
"""Create custom logger."""
custom_logger.remove()
custom_logger.add(stdout, colorize=True, format=formatter)
return custom_logger
def run_train(self, train, dev, args=None, verbose=True, eval_train=False):
args = args or self.args
if not os.path.isdir(self.dout):
os.makedirs(self.dout)
logger_loop = logger.bind(type='train_loop')
logger_loop.info('Using config\n{}'.format(args))
self.save_config()
flog = self.get_file('train.log')
fmetrics = open(self.get_file('train.metrics.jsonl'), 'wt')
fbestmetrics = self.get_file('train.best.json')
logger_loop.info('Logging to {}'.format(flog))
logger.add(flog, rotation='5 MB', mode='wt', level='INFO', format="{time} {extra[type]} -- {message}")
optimizer, scheduler = self.get_optimizer(train)
iteration = 0
start_epoch = 0
if args.resume:
metrics = self.load_save(fname=args.resume, optimizer=None if args.restart_optim else optimizer, scheduler=None if args.restart_optim or args.restart_scheduler else scheduler)
start_epoch = metrics['epoch']
iteration = metrics['iteration']
logger_loop.info('Resuming from {}'.format(args.resume))
logger_loop.info(pprint.pformat(metrics))
self.args = args
train_preds = None
best = {}
fbest = os.path.join(self.dout, 'best.tar')
for epoch in tqdm.trange(start_epoch, args.epoch, desc='epoch'):
logger_loop.info('Starting train epoch {}'.format(epoch))
loss = defaultdict(lambda: 0)
self.eval()
train = train.reset()
self.train()
timing = defaultdict(list)
for batch in train.batch(args.batch, shuffle=True, verbose=verbose, desc='train'):
start_time = time.time()
self.state.update({'iteration': iteration, 'epoch': epoch})
feat = self.featurize(batch)
out = self.forward(**feat)
forward_time = time.time()
loss_ = self.compute_loss(out, feat, batch)
loss_backprop = 0
if isinstance(loss_, dict):
for k, v in loss_.items():
loss['loss_' + k] += v.item() * len(batch)
loss_backprop += v
else:
loss['loss'] += loss_.item() * len(batch)
loss_backprop += loss_
iteration += len(batch)
loss_backprop.backward()
torch.nn.utils.clip_grad_norm_(self.parameters(), self.args.max_grad_norm)
old_params = copy.deepcopy(list(self.named_parameters()))
optimizer.step()
# running avg
new_params = dict(self.named_parameters())
interp = self.args.running_avg
for name, op in old_params:
new_params[name].data.copy_(interp*op.data + (1-interp)*new_params[name].data)
self.load_state_dict(new_params)
scheduler.step()
optimizer.zero_grad()
backward_time = time.time()
if not eval_train:
train_preds = train.accumulate_preds(train_preds, self.extract_preds(out, feat, batch))
timing['forward'].append(forward_time - start_time)
timing['backward'].append(backward_time - forward_time)
timing['batch'].append(backward_time - start_time)
if eval_train:
train_preds = self.run_pred(train, args, verbose=verbose, desc='train_pred')
metrics = {'epoch': epoch, 'iteration': iteration}
for k, v in loss.items():
metrics[k] = v / len(train)
for k, v in timing.items():
metrics['time_{}'.format(k)] = sum(v) / len(v)
metrics.update({'train_{}'.format(k): v for k, v in self.compute_metrics(train, train_preds).items()})
dev_preds = self.run_pred(dev, args, verbose=verbose, desc='dev_pred')
metrics.update({'dev_{}'.format(k): v for k, v in self.compute_metrics(dev, dev_preds).items()})
fmetrics.write(json.dumps(metrics) + '\n')
logger_loop.info('\n' + pprint.pformat(metrics))
if self.better(metrics, best):
best.update(metrics)
logger_loop.info('Found new best! Saving checkpoint')
self.save(metrics, optimizer, scheduler, fbest)
self.write_preds(dev, dev_preds, self.get_file('dev.preds.json'))
with open(fbestmetrics, 'wt') as f:
json.dump(metrics, f, indent=2)
logger_loop.info('Loading best checkpoint from {}'.format(fbest))
metrics = self.load_save(fname=fbest)
logger_loop.info(pprint.pformat(metrics))
fmetrics.close()
import time
from loguru import logger
from codelab_adapter_client import AdapterNode
from codelab_adapter_client.utils import get_or_create_node_logger_dir, install_requirement
from codelab_adapter_client.thing import AdapterThing
from codelab_adapter_client.utils import threaded
import cozmo
from cozmo.util import degrees, distance_mm, speed_mmps
# logger.warning(dict(os.environ))
# log for debug
node_logger_dir = get_or_create_node_logger_dir()
debug_log = str(node_logger_dir / "debug.log")
logger.add(debug_log, rotation="1 MB", level="DEBUG")
class CozmoProxy(AdapterThing):
def __init__(self, node_instance):
super().__init__(thing_name="Cozmo", node_instance=node_instance)
self.disconnect_flag = False
def _say_hi(self, robot):
robot.say_text("hi").wait_for_completed()
def list(self, timeout=5) -> list:
if self.thing:
# 检查是否连接正常
return ["Cozmo"]
try:
def structured_logging_test():
logger.add(get_log_file("structured"), serialize=True)
logger.debug(test_data)
CYCLE_TESTER = env.int('CYCLE_TESTER', 20)
# definition of getter cycle, it will get proxy every CYCLE_GETTER second
CYCLE_GETTER = env.int('CYCLE_GETTER', 100)
# definition of tester
TEST_URL = env.str('TEST_URL', 'http://www.baidu.com')
TEST_TIMEOUT = env.int('TEST_TIMEOUT', 10)
TEST_BATCH = env.int('TEST_BATCH', 20)
# TEST_HEADERS = env.json('TEST_HEADERS', {
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
# })
TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302])
# definition of api
API_HOST = env.str('API_HOST', '0.0.0.0')
API_PORT = env.int('API_PORT', 5555)
API_THREADED = env.bool('API_THREADED', True)
# flags of enable
ENABLE_TESTER = env.bool('ENABLE_TESTER', True)
ENABLE_GETTER = env.bool('ENABLE_GETTER', True)
ENABLE_SERVER = env.bool('ENABLE_SERVER', True)
logger.add(env.str('LOG_RUNTIME_FILE', 'runtime.log'),
level='DEBUG',
rotation='1 week',
retention='20 days')
logger.add(env.str('LOG_ERROR_FILE', 'error.log'),
level='ERROR',
rotation='1 week')
import argparse
import sys
import getpass
import json
import cgi
import datetime
from urllib.parse import unquote
D2L_BASEURL = "https://mycourses.rit.edu/"
# Not sure if this is unique to me, or just unique to RIT's tenant
OU = 6605
logger.remove()
logger.add(sys.stderr, level="INFO")
# basically, mkdir -p /blah/blah/blah
def mkdir_recursive(path):
try:
os.makedirs(path, exist_ok=True)
except Exception as e:
logger.error("Exception: {}".format(e))
exit(1)
def get_xfrs_token(page_html):
"""
Method to parse a D2L page to find the XSRF.Token. The token is returned as a string
:param page_html:
:return:
import os
import os.path
import shutil
import datetime
from loguru import logger
from pathlib import Path
import mongoengine as me
from me_models import Db_connect, Queue, State, Gphoto, SourceList
from utils import file_md5sum, config
from gphoto_upload import upload_to_gphotos
from drive_walk import GphotoSync
cfg = config()
logger.add("app.log", rotation="1 MB")
Db_connect()
# TODO: Change logging levels to eliminate most logging
def main():
QueueWorker()
print("Main Done")
"""
Queue maintenance runs continuously.
"""
class QueueWorker:
def __init__(self):
