def test_cross_dataset(config_file,test_dataset, **kwargs):
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k,v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
PersonReID_Dataset_Downloader('./datasets',cfg.DATASETS.NAMES)
_, _, _, num_classes = data_loader(cfg,cfg.DATASETS.NAMES)
PersonReID_Dataset_Downloader('./datasets',test_dataset)
_, val_loader, num_query, _ = data_loader(cfg,test_dataset)
re_ranking=cfg.RE_RANKING
if not re_ranking:
logger = make_logger("Reid_Baseline", cfg.OUTPUT_DIR,
cfg.DATASETS.NAMES+'->'+test_dataset)
logger.info("Test Results:")
else:
logger = make_logger("Reid_Baseline", cfg.OUTPUT_DIR,
cfg.DATASETS.NAMES+'->'+test_dataset+'_re-ranking')
logger.info("Re-Ranking Test Results:")
device = torch.device(cfg.DEVICE)
model = getattr(models, cfg.MODEL.NAME)(num_classes)
model.load(cfg.OUTPUT_DIR,cfg.TEST.LOAD_EPOCH)
model = model.eval()
all_feats = []
all_pids = []
all_camids = []
since = time.time()
for data in tqdm(val_loader, desc='Feature Extraction', leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
if device:
model.to(device)
images = images.to(device)
feats = model(images)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats,all_pids,all_camids,num_query,re_ranking)
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(r, cmc[r - 1]))
test_time = time.time() - since
logger.info('Testing complete in {:.0f}m {:.0f}s'.format(test_time // 60, test_time % 60))
def __init__(self):
self.DRIVER_PATH = './driver/chromedriver'
self.INSTA_ID = '*****@*****.**'
self.INSTA_PASSWD = 'gh767600'
self.image_dir = './images'
self.db = DatabaseHelper()
self.logger = make_logger('instagram')
self.load_driver()
def __init__(self):
self.logger = logger.make_logger('broker.log')
self.ctx = zmq.Context.instance()
self.mailbox = make_socket(self.ctx)
self.devs = set()
self.poller = zmq.Poller()
self.top = tk.Tk()
self.setup_ui()
self.mail_table = {}
self.ui_running = True
def __init__(self, name, **kwargs):
self.name = name
self.params = kwargs
self.running = False
self.logger = logger.make_logger(name + '.log')
self.ctx = zmq.Context.instance()
self.mailbox = make_socket(self.ctx, name)
self.poller = zmq.Poller()
self.cmd_queue = CommandQueue()
self.state = 'closed'
def feedback():
"""
A function handling user feedback
"""
if request.method == 'GET':
return redirect('/about')
elif request.method == 'POST':
log = make_logger('feedback.log')
text = request.form['feedback_text']
log(text)
return redirect('/')
def train(cfg):
# output
output_dir = cfg.OUTPUT_DIR
if os.path.exists(output_dir):
raise KeyError("Existing path: ", output_dir)
else:
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'config.yaml'), 'w') as f_out:
print(cfg, file=f_out)
# logger
logger = make_logger("project", output_dir, 'log')
# device
num_gpus = 0
if cfg.DEVICE == 'cuda':
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.DEVICE_ID
num_gpus = len(cfg.DEVICE_ID.split(','))
logger.info("Using {} GPUs.\n".format(num_gpus))
cudnn.benchmark = True
device = torch.device(cfg.DEVICE)
# data
train_loader, query_loader, gallery_loader, num_classes = make_loader(cfg)
# model
model = make_model(cfg, num_classes=num_classes)
if num_gpus > 1:
model = nn.DataParallel(model)
# solver
criterion = make_loss(cfg, num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
# do_train
trainer = Trainer(model=model,
optimizer=optimizer,
criterion=criterion,
logger=logger,
scheduler=scheduler,
device=device)
trainer.run(start_epoch=0,
total_epoch=cfg.SOLVER.MAX_EPOCHS,
train_loader=train_loader,
query_loader=query_loader,
gallery_loader=gallery_loader,
print_freq=cfg.SOLVER.PRINT_FREQ,
eval_period=cfg.SOLVER.EVAL_PERIOD,
out_dir=output_dir)
print('Done.')
def __init__(self):
conn_info = {
'host': '192.168.219.110',
'dbname': 'insta_matzip',
'user': '******',
'password': '******',
'port': '5432'
}
self.conn = psycopg2.connect(**conn_info)
self.conn.autocommit = True
self.cur = self.conn.cursor()
self.logger = make_logger('db')
def main(newspapers, rewrite):
# reads from urls.data, writes to database
# check if html exists
logger = make_logger("logger.log")
from collect_urls import main as collect_urls
# get all urls from urls.data
urls = collect_urls(num=-1,
newspapers=newspapers,
source="urls.data",
parse=False)
for url in urls:
parse_url(url, rewrite=rewrite, logger=logger)
def main(num, newspapers, source, parse):
logger = make_logger("logger.log")
logger.info(f"collecting {num} from {newspapers} from {source}")
home = TextFiles()
newspapers = get_newspapers_from_registry(newspapers)
print(newspapers)
collection = []
for paper in newspapers:
if source == "google":
urls = collect_from_google(num, paper, logger)
if logger:
logger.info(f"saving {len(urls)} to file")
home.write(urls, "urls.data", "a")
elif source == "urls.data":
urls = home.get("urls.data")
urls = urls.split("\n")
urls.remove("")
urls = [
u
for u in urls
if paper["newspaper_url"] in u
if paper["checker"](u, logger)
][:num]
if logger:
logger.info(f"loaded {len(urls)} urls from {source}")
collection.extend(urls)
collection = set(collection)
logger.info(f"collected {len(collection)} urls")
if parse:
for url in collection:
parse_url(url, rewrite=True, logger=logger)
return collection
def route_mobile():
"""
A function providing the route for mobile devices
POST args:
_from (str): place of departure
_to (str): place of arrival
when(str): 'now' | 'today' | 'tomorrow'
when_param(str): '%H:%M' datetime for reverse routing
device_id(str): mobile device id for logging
Returns:
_route (str): JSON-formatted string with route
"""
log = make_logger('route_mobile.log')
_from = request.form['_from']
_to = request.form['_to']
when = request.form['when']
when_param = request.form['when_param']
device_id = request.form['device_id']
log("{device_id} {_from} {_to}".format
(
device_id=device_id,
_from=_from,
_to=_to))
if when == 'now':
_route = calculate_route(_from, _to, datetime.now() + timedelta(minutes=10), "MOBILE")
else:
_date = datetime.now()
_time = [int(x) for x in when_param.split(':')]
_date = _date.replace(hour=_time[0], minute=_time[1])
if when == 'tomorrow':
_date += timedelta(days=1)
_route = calculate_route_reverse(_from, _to, _date)
js_route = json.dumps(_route, cls=DateTimeAwareJSONEncoder)
response = make_response(js_route)
response.headers['Content-Type'] = 'application/json; charset=utf-8'
return response
def train(config_file1, config_file2, **kwargs):
# 1. config
cfg.merge_from_file(config_file1)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
#cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log')
#logger.info("Using {} GPUS".format(1))
logger.info("Loaded configuration file {}".format(config_file1))
logger.info("Running with config:\n{}".format(cfg))
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
eval_period = cfg.SOLVER.EVAL_PERIOD
#device = torch.device(cfg.DEVICE)
epochs = cfg.SOLVER.MAX_EPOCHS
# 2. datasets
# Load the original dataset
#dataset_reference = init_dataset(cfg, cfg.DATASETS.NAMES )
dataset_reference = init_dataset(cfg, cfg.DATASETS.NAMES +
'_origin') #'Market1501_origin'
train_set_reference = ImageDataset(dataset_reference.train,
train_transforms)
train_loader_reference = DataLoader(train_set_reference,
batch_size=128,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
collate_fn=train_collate_fn)
#不用放到网络里，所以不用transform
# Load the one-shot dataset
train_loader, val_loader, num_query, num_classes = data_loader(
cfg, cfg.DATASETS.NAMES)
# 3. load the model and optimizer
model = getattr(models, cfg.MODEL.NAME)(num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
loss_fn = make_loss(cfg)
logger.info("Start training")
since = time.time()
if torch.cuda.device_count() > 1:
print("Use", torch.cuda.device_count(), 'gpus')
elif torch.cuda.device_count() == 1:
print("Use", torch.cuda.device_count(), 'gpu')
model = nn.DataParallel(model)
top = 0 # the choose of the nearest sample
top_update = 0 # the first iteration train 80 steps and the following train 40
train_time = 0 #1表示训练几次gan
bound = 1 #究竟训练几次，改成多次以后再说
lock = False
train_compen = 0
# 4. Train and test
for epoch in range(epochs):
running_loss = 0.0
running_acc = 0
count = 1
# get nearest samples and reset the model
if top_update < 80:
train_step = 80
#重新gan生成的图像第一次是否需要训练80次，看看是否下一次输入的图片变少了吧
else:
train_step = 40
#if top_update % train_step == 0:
if top_update % train_step == 0 and train_compen == 0:
print("top: ", top)
#作者原来的实验top取到41，这里折中(是否要折中也是个实验测试的点)
#if 1==1:
if top >= 8 and train_time < bound:
train_compen = (top - 1) * 40 + 80
#build_image(A,train_loader_reference,train_loader)
train_time += 1
#gan的训练模式
mode = 'train'
retrain(mode)
#gan生成图像到原来数据集
produce()
cfg.merge_from_file(config_file2)
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log')
logger.info(
"Loaded configuration file {}".format(config_file2))
logger.info("Running with config:\n{}".format(cfg))
dataset_reference = init_dataset(
cfg, cfg.DATASETS.NAMES + '_origin') #'Market1501_origin'
train_set_reference = ImageDataset(dataset_reference.train,
train_transforms)
train_loader_reference = DataLoader(
train_set_reference,
batch_size=128,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
collate_fn=train_collate_fn)
dataset_ref = init_dataset(cfg, cfg.DATASETS.NAMES +
'_ref') #'Market1501_origin'
train_set_ref = ImageDataset(dataset_ref.train,
train_transforms)
train_loader_ref = DataLoader(
train_set_ref,
batch_size=128,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
collate_fn=train_collate_fn)
lock = True
if lock == True:
A, path_labeled = PSP2(model, train_loader_reference,
train_loader, train_loader_ref, top,
logger, cfg)
lock = False
else:
A, path_labeled = PSP(model, train_loader_reference,
train_loader, top, logger, cfg)
#vis = len(train_loader_reference.dataset)
#A= torch.ones(vis, len(train_loader_reference.dataset))
#build_image(A,train_loader_reference,train_loader)
top += cfg.DATALOADER.NUM_JUMP
model = getattr(models, cfg.MODEL.NAME)(num_classes)
model = nn.DataParallel(model)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
A_store = A.clone()
top_update += 1
for data in tqdm(train_loader, desc='Iteration', leave=False):
model.train()
images, labels_batch, img_path = data
index, index_labeled = find_index_by_path(img_path,
dataset_reference.train,
path_labeled)
images_relevant, GCN_index, choose_from_nodes, labels = load_relevant(
cfg, dataset_reference.train, index, A_store, labels_batch,
index_labeled)
# if device:
model.to(device)
images = images_relevant.to(device)
scores, feat = model(images)
del images
loss = loss_fn(scores, feat, labels.to(device), choose_from_nodes)
optimizer.zero_grad()
loss.backward()
optimizer.step()
count = count + 1
running_loss += loss.item()
running_acc += (scores[choose_from_nodes].max(1)[1].cpu() ==
labels_batch).float().mean().item()
scheduler.step()
# for model save if you need
# if (epoch+1) % checkpoint_period == 0:
# model.cpu()
# model.save(output_dir,epoch+1)
# Validation
if (epoch + 1) % eval_period == 0:
all_feats = []
all_pids = []
all_camids = []
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
model.to(device)
images = images.to(device)
feats = model(images)
del images
all_feats.append(feats.cpu())
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids, num_query)
logger.info("Validation Results - Epoch: {}".format(epoch + 1))
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10, 20]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(
r, cmc[r - 1]))
if train_compen > 0:
train_compen -= 1
time_elapsed = time.time() - since
logger.info('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
logger.info('-' * 10)
Line 139:20140117_020001:(14:E:\Logs\1234\) skip [20140117]E:\Logs\1234\StartupException_20140117_003531_455.log
BAD
Line 478:20140117_020005:(24:E:\var\Logs\1234\ExceptionLogBackup\) Starting
GOOD
Line 3302:20140117_030522:(20:E:\Logs\Wedge\Auto\) Failed deleting file
[20131227_Task20_Auto.3]E:\Logs\Wedge\Auto\new_path_summary.log.12-27-2013.log.System.IO.IOException:
The process cannot access the file 'E:\Logs\Wedge\Auto\new_path_summary.log.12-27-2013.log' because it is being used by another process.
Line 49:System.Web.Services.Protocols.SoapException: Server was unable to process request. ---> Object reference not set to an instance of an object.
matches found:1
"""
log = logger.make_logger('main', 'archiverUpdate.log', '.', 10).log
def runVmBFA():
log(10, 'start')
xml = subprocess.check_output("Topology VmBox %")
log(10, 'xml returned')
status, vmb_obj = VmBoxList.obj_wrapper(xml)
log(10, 'vmb_obj returned')
pool = multiprocessing.Pool(10)
pool.map(FindArchiver(), vmb_obj.VMBox)
class FindArchiver(object):
fa_re_obj = re.compile(r'filearchive.exe', re.IGNORECASE)
try:
import logger
logger = logger.make_logger('pserver-test', debug=False, colored=False)
except ImportError:
import logging
logging.basicConfig(level = logging.DEBUG)
logger = logging#.getLogger()
import subprocess
import os
import sys
from datetime import datetime
import time
import yaml
from logger import make_logger
LOGGER = make_logger(sys.stderr, "gpu_protector")
def main():
kill_temperature = int(sys.argv[1])
with open("miner_conf.yaml", 'r') as f:
miner_name_list = yaml.load(f, Loader=yaml.FullLoader)
LOGGER.info("Miner Program List: {}".format(miner_name_list))
print("Miner Program List: {}".format(miner_name_list))
process = subprocess.Popen(
"nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
output = process.communicate()[0]
for i, line in enumerate(output.splitlines(), 0):
gputemp = int(line.decode())
LOGGER.info("GPU #{} Temperature = {}".format(i, gputemp))
print("GPU #{} Temperature = {}".format(i, gputemp))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
A module providing functionality for calculating routes
"""
from datetime import datetime
from datetime import timedelta
from route_bus import get_nearest_bus
from route_train import get_nearest_train
from route_subway import get_nearest_subway
from route_onfoot import get_nearest_onfoot
from logger import make_logger
ROUTE_LOG = make_logger('routing.log')
JSON_LOG = make_logger('json.log')
REVERSE_ROUTE_LOG = make_logger('reverse_routing.log')
#: list of dormitories
DORMS = {
'dubki': 'Дубки',
'odintsovo': 'Одинцово'
}
#: list of education campuses
EDUS = {
'aeroport': 'Кочновский проезд (метро Аэропорт)',
'strogino': 'Строгино',
'myasnitskaya': 'Мясницкая (метро Лубянка)',
'vavilova': 'Вавилова (метро Ленинский проспект)',
import urllib3
import requests
import http
from functools import wraps
from logger import make_logger
import os
logger = make_logger(__name__, 'dataGetter_log')
def connection_exception(f):
@wraps(f)
def call(*args, **kwargs):
result = None
try:
result = f(*args, **kwargs)
except urllib3.response.ProtocolError as e:
logger.error(f'[urllib3] {f.__name__}', e)
except http.client.IncompleteRead as e:
logger.error(f'[http] {f.__name__}', e)
except requests.models.ChunkedEncodingError as e:
logger.error(f'[requests] {f.__name__}', e)
except BaseException as e:
logger.error(
'some Exception happend when send and receiving data. %s ', e)
finally:
# if os.environ.get('DEBUG_DG_API'):
# print("==========>")
# print(f)
# print("==========>")
"""
This is only used in db_init and db_init now is largely
deprecated.
This garbage really shows what will happen if you try to do
concurrency without some study unfront.
"""
from threading import Thread, Lock
from queue import Queue
from typing import TypeVar, Iterable, Callable, List, cast
from itertools import islice
from .generator_chunks import chunks
from logger import make_logger
logger = make_logger('concurrent_fetch', 'concurrent_fetch_log')
logger.propagate = False
T = TypeVar("T")
LazyBox = Iterable # iter with only one element. for lazy eval
JobBox = LazyBox[T]
def thread_fetcher(jobs: Iterable[Callable[[], T]],
max_thread: int,
fetcher: Callable[[Queue, Callable[[], T], Lock], None],
consumer: Callable[[Queue], None] = lambda q: None,
before_consume_hook: Callable = lambda: None,
after_consume_hook: Callable = lambda: None) -> None:
"""
Threaded worker for data fetching
# Command: python miner_watchdog.py kai_test_miner 15 ETH multi debug
# Eth speed: 46.754 MH/s
OK = '\033[36m'
FAIL = '\033[41m'
LOG = '\033[32m'
WARNING = '\033[33m'
ENDC = '\033[0m'
READ_FLAG = [True]
current_pid = None
miner_stdout_buffer = []
LOGGER = make_logger(sys.stderr, "coin-watchdog")
def get_most_profitable_coin(coin_pos, miner_dict, default_coin):
try:
scope = ['https://spreadsheets.google.com/feeds']
creds = ServiceAccountCredentials.from_json_keyfile_name(
'key3.json', scope)
gc = gspread.authorize(creds)
wks = gc.open_by_url(
"https://docs.google.com/spreadsheets/d/12G_XdpgLKY_nb3zYI1BWfncjMJBcqVROkHoJcXO-JcE"
).worksheet("Coin Switch")
coin = wks.acell(coin_pos).value
except Exception as e:
print(
FAIL,
def train(config_file, resume=False, iteration=10, STEP=4, **kwargs):
"""
Parameter
---------
resume : bool
If true, continue the training and append logs to the previous log.
iteration : int
number of loops to test Random Datasets.
STEP : int
Number of steps to train the discriminator per batch
"""
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
# [PersonReID_Dataset_Downloader('./datasets', name) for name in cfg.DATASETS.NAMES]
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log', resume)
if not resume:
logger.info("Using {} GPUS".format(1))
logger.info("Loaded configuration file {}".format(config_file))
logger.info("Running with config:\n{}".format(cfg))
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
eval_period = cfg.SOLVER.EVAL_PERIOD
output_dir = cfg.OUTPUT_DIR
device = torch.device(cfg.DEVICE)
epochs = cfg.SOLVER.MAX_EPOCHS
sources = cfg.DATASETS.SOURCE
target = cfg.DATASETS.TARGET
pooling = cfg.MODEL.POOL
last_stride = cfg.MODEL.LAST_STRIDE
# tf_board_path = os.path.join(output_dir, 'tf_runs')
# if os.path.exists(tf_board_path):
# shutil.rmtree(tf_board_path)
# writer = SummaryWriter(tf_board_path)
gan_d_param = cfg.MODEL.D_PARAM
gan_g_param = cfg.MODEL.G_PARAM
class_param = cfg.MODEL.CLASS_PARAM
"""Set up"""
train_loader, _, _, num_classes = data_loader(cfg,
cfg.DATASETS.SOURCE,
merge=cfg.DATASETS.MERGE)
num_classes_train = [
data_loader(cfg, [source], merge=False)[3]
for source in cfg.DATASETS.SOURCE
]
# based on input datasets
bias = (max(num_classes_train)) / np.array(num_classes_train)
bias = bias / bias.sum() * 5
discriminator_loss = LabelSmoothingLoss(len(sources),
weights=bias,
smoothing=0.1)
minus_generator_loss = LabelSmoothingLoss(len(sources),
weights=bias,
smoothing=0.)
classification_loss = LabelSmoothingLoss(num_classes, smoothing=0.1)
from loss.triplet_loss import TripletLoss
triplet = TripletLoss(cfg.SOLVER.MARGIN)
triplet_loss = lambda feat, labels: triplet(feat, labels)[0]
module = getattr(generalizers, cfg.MODEL.NAME)
D = getattr(module, 'Generalizer_D')(len(sources))
G = getattr(module, 'Generalizer_G')(num_classes, last_stride, pooling)
if resume:
checkpoints = get_last_stats(output_dir)
D.load_state_dict(torch.load(checkpoints[str(type(D))]))
G.load_state_dict(torch.load(checkpoints[str(type(G))]))
if device: # must be done before the optimizer generation
D.to(device)
G.to(device)
discriminator_optimizer = Adam(D.parameters(),
lr=cfg.SOLVER.BASE_LR,
weight_decay=cfg.SOLVER.WEIGHT_DECAY)
generator_optimizer = Adam(G.parameters(),
lr=cfg.SOLVER.BASE_LR,
weight_decay=cfg.SOLVER.WEIGHT_DECAY)
discriminator_scheduler = make_scheduler(cfg, discriminator_optimizer)
generator_scheduler = make_scheduler(cfg, generator_optimizer)
base_epo = 0
if resume:
discriminator_optimizer.load_state_dict(
torch.load(checkpoints['D_opt']))
generator_optimizer.load_state_dict(torch.load(checkpoints['G_opt']))
discriminator_scheduler.load_state_dict(
torch.load(checkpoints['D_sch']))
generator_scheduler.load_state_dict(torch.load(checkpoints['G_sch']))
base_epo = checkpoints['epo']
# Modify the labels:
# RULE:
# according to the order of names in cfg.DATASETS.NAMES, add base numebr
since = time.time()
if not resume:
logger.info("Start training")
batch_count = 0
STEP = 4
Best_R1s = [0, 0, 0, 0]
Benchmark = [69.6, 43.7, 59.4, 78.2]
for epoch in range(epochs):
# anneal = sigmoid(annealing_base + annealing_factor*(epoch+base_epo))
anneal = max(1 - (1 / 80 * epoch), 0)
count = 0
running_g_loss = 0.
running_source_loss = 0.
running_class_acc = 0.
running_acc_source = 0.
running_class_loss = 0.
reset()
for data in tqdm(train_loader, desc='Iteration', leave=False):
# NOTE: zip ensured the shortest dataset dominates the iteration
D.train()
G.train()
images, labels, domains = data
if device:
D.to(device)
G.to(device)
images, labels, domains = images.to(device), labels.to(
device), domains.to(device)
"""Start Training D"""
feature_vec, scores, gan_vec = G(images)
for param in G.parameters():
param.requires_grad = False
for param in D.parameters():
param.requires_grad = True
for _ in range(STEP):
discriminator_optimizer.zero_grad()
pred_domain = D(
[v.detach()
for v in gan_vec] if isinstance(gan_vec, list) else
gan_vec.detach()) # NOTE: Feat output! Not Probability!
d_losses, accs = discriminator_loss(pred_domain,
domains,
compute_acc=True)
d_source_loss = d_losses.mean()
d_source_acc = accs.float().mean().item()
d_loss = d_source_loss
w_d_loss = anneal * d_loss * gan_d_param
w_d_loss.backward()
discriminator_optimizer.step()
"""Start Training G"""
for param in D.parameters():
param.requires_grad = False
for param in G.parameters():
param.requires_grad = True
generator_optimizer.zero_grad()
g_loss = -1. * minus_generator_loss(D(gan_vec), domains).mean()
class_loss = classification_loss(scores, labels).mean()
tri_loss = triplet_loss(feature_vec, labels)
class_loss = class_loss * cfg.SOLVER.LAMBDA1 + tri_loss * cfg.SOLVER.LAMBDA2
w_regularized_g_loss = anneal * gan_g_param * g_loss + class_param * class_loss
w_regularized_g_loss.backward()
generator_optimizer.step()
"""Stop training"""
running_g_loss += g_loss.item()
running_source_loss += d_source_loss.item()
running_acc_source += d_source_acc # TODO: assume all batches are the same size
running_class_loss += class_loss.item()
class_acc = (scores.max(1)[1] == labels).float().mean().item()
running_class_acc += class_acc
# writer.add_scalar('D_loss', d_source_loss.item(), batch_count)
# writer.add_scalar('D_acc', d_source_acc, batch_count)
# writer.add_scalar('G_loss', g_loss.item(), batch_count)
# writer.add_scalar('Class_loss', class_loss.item(), batch_count)
# writer.add_scalar('Class_acc', class_acc, batch_count)
torch.cuda.empty_cache()
count = count + 1
batch_count += 1
# if count == 10:break
logger.info(
"Epoch[{}] Iteration[{}] Loss: [G] {:.3f} [D] {:.3f} [Class] {:.3f}, Acc: [Class] {:.3f} [D] {:.3f}, Base Lr: {:.2e}"
.format(epoch + base_epo + 1, count, running_g_loss / count,
running_source_loss / count, running_class_loss / count,
running_class_acc / count, running_acc_source / count,
generator_scheduler.get_lr()[0]))
generator_scheduler.step()
discriminator_scheduler.step()
if (epoch + base_epo + 1) % checkpoint_period == 0:
G.cpu()
G.save(output_dir, epoch + base_epo + 1)
D.cpu()
D.save(output_dir, epoch + base_epo + 1)
torch.save(
generator_optimizer.state_dict(),
os.path.join(output_dir,
'G_opt_epo' + str(epoch + base_epo + 1) + '.pth'))
torch.save(
discriminator_optimizer.state_dict(),
os.path.join(output_dir,
'D_opt_epo' + str(epoch + base_epo + 1) + '.pth'))
torch.save(
generator_scheduler.state_dict(),
os.path.join(output_dir,
'G_sch_epo' + str(epoch + base_epo + 1) + '.pth'))
torch.save(
discriminator_scheduler.state_dict(),
os.path.join(output_dir,
'D_sch_epo' + str(epoch + base_epo + 1) + '.pth'))
# Validation
if (epoch + base_epo + 1) % eval_period == 0:
# Validation on Target Dataset
for target in cfg.DATASETS.TARGET:
mAPs = []
cmcs = []
for i in range(iteration):
set_seeds(i)
_, val_loader, num_query, _ = data_loader(cfg, (target, ),
merge=False,
verbose=False)
all_feats = []
all_pids = []
all_camids = []
since = time.time()
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
G.eval()
with torch.no_grad():
images, pids, camids = data
if device:
G.to(device)
images = images.to(device)
feats = G(images)
feats /= feats.norm(dim=-1, keepdim=True)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids,
num_query)
mAPs.append(mAP)
cmcs.append(cmc)
mAP = np.mean(np.array(mAPs))
cmc = np.mean(np.array(cmcs), axis=0)
mAP_std = np.std(np.array(mAPs))
cmc_std = np.std(np.array(cmcs), axis=0)
logger.info("Validation Results: {} - Epoch: {}".format(
target, epoch + 1 + base_epo))
logger.info("mAP: {:.1%} (std: {:.3%})".format(mAP, mAP_std))
for r in [1, 5, 10]:
logger.info(
"CMC curve, Rank-{:<3}:{:.1%} (std: {:.3%})".format(
r, cmc[r - 1], cmc_std[r - 1]))
# Record Best
if (epoch + base_epo + 1) > 60 and ((epoch + base_epo + 1) % 5 == 1 or
(epoch + base_epo + 1) % 5 == 2):
# Validation on Target Dataset
R1s = []
for target in cfg.DATASETS.TARGET:
mAPs = []
cmcs = []
for i in range(iteration):
set_seeds(i)
_, val_loader, num_query, _ = data_loader(cfg, (target, ),
merge=False,
verbose=False)
all_feats = []
all_pids = []
all_camids = []
since = time.time()
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
G.eval()
with torch.no_grad():
images, pids, camids = data
if device:
G.to(device)
images = images.to(device)
feats = G(images)
feats /= feats.norm(dim=-1, keepdim=True)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids,
num_query)
mAPs.append(mAP)
cmcs.append(cmc)
mAP = np.mean(np.array(mAPs))
cmc = np.mean(np.array(cmcs), axis=0)
R1 = cmc[0]
R1s.append(R1)
if (np.array(R1s) > np.array(Best_R1s)).all():
logger.info("Best checkpoint at {}: {}".format(
str(epoch + base_epo + 1),
', '.join([str(s) for s in R1s])))
Best_R1s = R1s
G.cpu()
G.save(output_dir, -1)
D.cpu()
D.save(output_dir, -1)
torch.save(
generator_optimizer.state_dict(),
os.path.join(output_dir, 'G_opt_epo' + str(-1) + '.pth'))
torch.save(
discriminator_optimizer.state_dict(),
os.path.join(output_dir, 'D_opt_epo' + str(-1) + '.pth'))
torch.save(
generator_scheduler.state_dict(),
os.path.join(output_dir, 'G_sch_epo' + str(-1) + '.pth'))
torch.save(
discriminator_scheduler.state_dict(),
os.path.join(output_dir, 'D_sch_epo' + str(-1) + '.pth'))
else:
logger.info("Rank 1 results: {}".format(', '.join(
[str(s) for s in R1s])))
time_elapsed = time.time() - since
logger.info('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
logger.info('-' * 10)
import logger
gmaps_logger = logger.make_logger("gmaps_logger", "Gmaps_logger.log")
import requests
try:
import credentials
except (ModuleNotFoundError):
gmaps_logger.debug("Wrapper startup - no credentials file")
import logging
import datetime
import json
#wrapper for making requests to the google maps developer api
#can pass the api key in a "credentials.py" file or as a value
class Gmaps():
def __init__(self, key=None):
if not key:
#checks if a key was passed in a credentials file or a value
#if it wasnt - raises an error
try:
self.api_key = credentials.api_key
except (NameError):
gmaps_logger.debug(
"Authontication error - No credentials file and no key passed"
)
raise
else:
self.api_key = key
#return all places in a circle around the center coordinate with the "radius" radius, optionally only return a single type of place
def get_places_nearby(self,
#!/home/kyle/anaconda3/bin/python
import zmq
from MsgID import gen_id
import names
import time
import random
import logger
import errno
import tkinter as tk
NBR_DEVS = names.NBR_DEVS
app_log = logger.make_logger('broker.log')
"""
mail_table is a dictionary
the key is a uuid generated by uuid.uuid4(), used to uniquely identify messages
the value for each key is a tuple (from_addr, to_addr, msg)
devs is a python set which contain bytes representation of names
"""
class Broker():
def __init__(self):
self.logger = logger.make_logger('broker.log')
self.ctx = zmq.Context.instance()
self.mailbox = make_socket(self.ctx)
self.devs = set()
self.poller = zmq.Poller()
self.top = tk.Tk()
def test_random_datasets(config_file,
iteration=10,
model_type="generalizer",
**kwargs):
if model_type == "normal":
from config.default_multi_domain import _C as cfg
elif model_type == "generalizer":
from config.default_multi_domain import _C as cfg
else:
raise ValueError("Model type can only be normal or generalizer.")
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
# PersonReID_Dataset_Downloader('./datasets',cfg.DATASETS.NAMES)
_, _, _, num_classes = data_loader(cfg,
cfg.DATASETS.SOURCE,
merge=cfg.DATASETS.MERGE)
re_ranking = cfg.RE_RANKING
device = torch.device(cfg.DEVICE)
if model_type == "generalizer":
module = getattr(generalizers, cfg.MODEL.NAME)
model = getattr(module,
'Generalizer_G')(num_classes, cfg.MODEL.LAST_STRIDE,
cfg.MODEL.POOL)
checkpoints = get_last_stats(cfg.OUTPUT_DIR)
model_dict = torch.load(checkpoints[str(type(model))])
model.load_state_dict(model_dict)
elif model_type == "normal":
model = getattr(models,
cfg.MODEL.NAME)(num_classes, cfg.MODEL.LAST_STRIDE,
cfg.MODEL.POOL)
checkpoints = get_last_stats(cfg.OUTPUT_DIR, [cfg.MODEL.NAME])
model_dict = torch.load(checkpoints[cfg.MODEL.NAME])
model.load_state_dict(model_dict)
model = model.eval()
if not re_ranking:
logger = make_logger("Reid_Baseline", cfg.OUTPUT_DIR,
'epo' + str(checkpoints['epo']))
logger.info("Test Results:")
else:
logger = make_logger("Reid_Baseline", cfg.OUTPUT_DIR,
'epo' + str(checkpoints['epo']) + '_re-ranking')
logger.info("Re-Ranking Test Results:")
for test_dataset in cfg.DATASETS.TARGET:
mAPs = []
cmcs = []
for i in range(iteration):
set_seeds(i)
_, val_loader, num_query, _ = data_loader(cfg, (test_dataset, ),
merge=False)
all_feats = []
all_pids = []
all_camids = []
since = time.time()
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
if device:
model.to(device)
images = images.to(device)
feats = model(images)
feats /= feats.norm(dim=-1, keepdim=True)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids, num_query,
re_ranking)
mAPs.append(mAP)
cmcs.append(cmc)
mAP = np.mean(np.array(mAPs))
cmc = np.mean(np.array(cmcs), axis=0)
mAP_std = np.std(np.array(mAPs))
cmc_std = np.std(np.array(cmcs), axis=0)
logger.info("mAP: {:.1%} (std: {:.3%})".format(mAP, mAP_std))
for r in [1, 5, 10]:
logger.info("CMC curve, Rank-{:<3}:{:.1%} (std: {:.3%})".format(
r, cmc[r - 1], cmc_std[r - 1]))
test_time = time.time() - since
logger.info('Testing complete in {:.0f}m {:.0f}s'.format(
test_time // 60, test_time % 60))
from ..apis.xiaomiGetter import ResourceResponse
from timeutils.time import date_range_iter
from timeutils.time import str_to_datetime
from timeutils.time import timestamp_setdigits
from .tokenManager import TokenManager
from .dataType import Device
from .dataType import Location
from .dataType import Spot
from .dataType import SpotData
from .dataType import SpotRecord
from .dataType import WrongDidException
from .dataType import device_check
from .dataType import DataSource
from .dataType import RecordThunkIter
logger = make_logger('dataMidware', 'dataGetter_log')
logger.propagate = False
"""
some of the device models are ignored. Largely because the are
just gateway for gathering data from its child devices.
"""
deviceModels: Dict = {
'lumi.acpartner.v3': ["on_off_status", "cost_energy"], # AC
'lumi.gateway.aq1': [], # ignore
'lumi.plug.v1': ["plug_status", "cost_energy"], # AC
'lumi.sensor_ht.v1': ["humitidy_value", "temperature_value"],
'lumi.sensor_magnet.v2': ["magnet_status"],
'lumi.sensor_motion.v2': [] # ignore
}
def train(config_file, **kwargs):
# 1. config
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log')
logger.info("Using {} GPUS".format(1))
logger.info("Loaded configuration file {}".format(config_file))
logger.info("Running with config:\n{}".format(cfg))
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
eval_period = cfg.SOLVER.EVAL_PERIOD
device = torch.device(cfg.DEVICE)
epochs = cfg.SOLVER.MAX_EPOCHS
# 2. datasets
# Load the original dataset
dataset_reference = init_dataset(cfg, cfg.DATASETS.NAMES +
'_origin') #'Market1501_origin'
train_set_reference = ImageDataset(dataset_reference.train,
train_transforms)
train_loader_reference = DataLoader(train_set_reference,
batch_size=128,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
collate_fn=train_collate_fn)
# Load the one-shot dataset
train_loader, val_loader, num_query, num_classes = data_loader(
cfg, cfg.DATASETS.NAMES)
# 3. load the model and optimizer
model = getattr(models, cfg.MODEL.NAME)(num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
loss_fn = make_loss(cfg)
logger.info("Start training")
since = time.time()
top = 0 # the choose of the nearest sample
top_update = 0 # the first iteration train 80 steps and the following train 40
# 4. Train and test
for epoch in range(epochs):
running_loss = 0.0
running_acc = 0
count = 1
# get nearest samples and reset the model
if top_update < 80:
train_step = 80
else:
train_step = 40
if top_update % train_step == 0:
print("top: ", top)
A, path_labeled = PSP(model, train_loader_reference, train_loader,
top, cfg)
top += cfg.DATALOADER.NUM_JUMP
model = getattr(models, cfg.MODEL.NAME)(num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
A_store = A.clone()
top_update += 1
for data in tqdm(train_loader, desc='Iteration', leave=False):
model.train()
images, labels_batch, img_path = data
index, index_labeled = find_index_by_path(img_path,
dataset_reference.train,
path_labeled)
images_relevant, GCN_index, choose_from_nodes, labels = load_relevant(
cfg, dataset_reference.train, index, A_store, labels_batch,
index_labeled)
# if device:
model.to(device)
images = images_relevant.to(device)
scores, feat = model(images)
del images
loss = loss_fn(scores, feat, labels.to(device), choose_from_nodes)
optimizer.zero_grad()
loss.backward()
optimizer.step()
count = count + 1
running_loss += loss.item()
running_acc += (scores[choose_from_nodes].max(1)[1].cpu() ==
labels_batch).float().mean().item()
scheduler.step()
# for model save if you need
# if (epoch+1) % checkpoint_period == 0:
# model.cpu()
# model.save(output_dir,epoch+1)
# Validation
if (epoch + 1) % eval_period == 0:
all_feats = []
all_pids = []
all_camids = []
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
model.to(device)
images = images.to(device)
feats = model(images)
del images
all_feats.append(feats.cpu())
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids, num_query)
logger.info("Validation Results - Epoch: {}".format(epoch + 1))
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10, 20]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(
r, cmc[r - 1]))
time_elapsed = time.time() - since
logger.info('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
logger.info('-' * 10)
#!/usr/bin/python
import os
import psycopg2
import json
import time
from filters_json import filter_list as FilterMap
from logger import make_logger
import inspect
logger = make_logger(inspect.stack()[0][1], 'retoracle.log')
QUERY_STRINGS = {}
DB_CONFIG = {}
NUM_TWEETS_NEED_SA = """\
SELECT COUNT(DISTINCT tweet_id)
FROM
(SELECT tweet_id FROM tweets
EXCEPT
SELECT tweet_id FROM tweet_sent) as intrsct;
"""
GET_TWEET_BATCH_NEED_SA = """SELECT tweet_id, tweet_text
FROM tweets
WHERE tweet_id
IN
(SELECT tweet_id FROM tweets EXCEPT select tweet_id FROM tweet_sent)
LIMIT %s;
"""
import Gmaps
import logger
import time
import json
circle_logger = logger.make_logger("circle_logger", "Circle_logger.log")
#add types if u finish!
class Circle():
def __init__(self, center, radius, type=None):
self.center = center
self.radius = radius
self.type = type
#try to create a gmaps object - if failed raise a value error
try:
self.Gmaps = Gmaps.Gmaps()
except (NameError):
raise (ValueError(
"Gmaps authentication error - no key or credentials file"))
#attempt to find all places in circle
self.my_places = self.find_all_places_in_circle()
#self.my_places = self.my_places["results"]
#if function returns string - return the error
if isinstance(self.my_places, dict):
# import sys import json import time import sys import boto import sql_queries as sql_q from boto.s3.key import Key import boto.emr from boto.emr.step import StreamingStep import SA_Mapper from SentimentAnalysis import agg_sent from logger import make_logger import inspect logger = make_logger(inspect.stack()[0][1], 'retoracle.log') # At most the worker will check for more Tweets to # analyze every MIN_EXECUTION_PERIOD seconds MIN_EXECUTION_PERIOD = 5.0 # If the number of Tweets to analyze is > EMR_THRESHOLD # Then this worker will use Hadoop to do SA # But only is ALLOW_EMR is also True EMR_THRESHOLD = 1000 # When creating a Hadoop job, whats the max batch size to aim for? MAX_BATCH_SIZE = 5000 # Allow this worker to spin up EMR (Amazon's Hadoop) # jobs for up to BATCH_SIZE batches of Tweets that need SA?
def train(opt):
train_dataset = VOCDetection(transform=SSDAugmentation(
opt.DATASETS.MIN_DIM, opt.DATASETS.MEANS),
opt=opt)
test_dataset = VOCDetection(['test'],
BaseTransform(300, opt.DATASETS.MEANS),
VOCAnnotationTransform(),
opt=opt)
ssd_net = build_ssd('train', opt.DATASETS.MIN_DIM, opt.DATASETS.NUM_CLS)
net = ssd_net
# logger
logger = make_logger("project", opt.OUTPUT_DIR, 'log')
if len(opt.DEVICE_ID) > 1:
net = torch.nn.DataParallel(ssd_net)
cudnn.benchmark = True
if opt.MODEL.RESUM:
print('Resuming training, loading {}...'.format(opt.MODEL.RESUM))
ssd_net.load_weights(opt.MODEL.RESUM)
else:
vgg_weights = torch.load(
os.path.join(opt.MODEL.BACKBONE_WEIGHTS, opt.MODEL.BACKBONE))
print('Loading base network...')
ssd_net.vgg.load_state_dict(vgg_weights)
if opt.DEVICE:
net = net.cuda()
if not opt.MODEL.RESUM:
print('Initializing backbone_weights...')
# initialize newly added layers' backbone_weights with xavier method
ssd_net.extras.apply(weights_init)
ssd_net.loc.apply(weights_init)
ssd_net.conf.apply(weights_init)
optimizer = optim.SGD(net.parameters(),
lr=opt.SOLVER.BASE_LR,
momentum=opt.SOLVER.MOMENTUM,
weight_decay=opt.SOLVER.WEIGHT_DECAY)
criterion = MultiBoxLoss(opt.DATASETS.NUM_CLS, 0.5, True, 0, True, 3, 0.5,
False, opt.DEVICE)
epoch_size = len(train_dataset) // opt.DATALOADER.BATCH_SIZE
train_loader = data.DataLoader(train_dataset,
batch_size=opt.DATALOADER.BATCH_SIZE,
num_workers=opt.DATALOADER.NUM_WORKERS,
shuffle=True,
collate_fn=detection_collate,
pin_memory=True)
# device = torch.device("cuda")
trainer = Trainer(net,
optimizer,
criterion,
logger,
device=None,
scheduler=None)
trainer.run(
opt=opt,
train_loader=train_loader, # dataloader
test_dataset=test_dataset, # dataset
epoch_size=epoch_size)
import requests
from hashlib import md5, sha1
from typing import Dict, Optional, Tuple, TypedDict, List, cast
from operator import itemgetter
import urllib3
import http
import urllib.parse
from datetime import datetime as dt
from timeutils.time import currentTimestamp
import json
from .exceptions import connection_exception
from logger import make_logger
logger = make_logger('xiaomiGetter', 'dataGetter_log')
"""
information for authentication, used for establishing the connecetion
with xiaomi platform.
"""
AuthData = (
TypedDict(
'AuthData',
{
'account': str,
'password': str,
'appId': str, # called client_id in json request.
# called client_secret in json request.
'appKey': str,
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
from flask_login import LoginManager
login_manager = LoginManager()
login_manager.login_view = 'auth.login'
from .caching import CacheInstance
global_cache = CacheInstance() # create cache instance here.
if db is not None:
# Scheduler depends on db.
from .dataGetter.dataloader.Scheduler import UpdateScheduler
scheduler = UpdateScheduler()
logger = make_logger('app', 'app_log', logging.DEBUG)
logger.warning('initializing app')
def create_app(config_name: str, with_scheduler: bool = True) -> Flask:
""" application factory function."""
app = Flask(__name__)
# load config
app.config.from_object(config[config_name])
config[config_name].init_app(app)
moment.init_app(app)
db.init_app(app)
login_manager.init_app(app)
if with_scheduler:
import CurrentState as cs
import CurrentThreshold as ct
import logger
import XMLSerializers.XMLAdapter as xa
import Alert as a
from datetime import datetime as dt
log = logger.make_logger('main', "BatchCheck.log", r"E:\BatchMon\Check", 10).log
def make_FI_list(dbs_obj):
FI_list = [row.attrs['source_DBName'].upper() for row in dbs_obj.row
if row.attrs['source_DBName'] not in ('TestDB2', 'TestDB3')
and row.attrs['source_is_read_only'] == '0']
return list(set(FI_list))
def process_batchCheck():
width = 3.7
log(20, 'Starting process_batchCheck, Config: width:{}'.format(width))
with xa.XMLAdapter() as xa_obj:
with open('E:\BatchMon\Alert\{}-alert.txt'.format(dt.now().strftime('%Y-%m-%dT%H%M%S')), 'w') as _f:
dbs_obj = xa_obj.get_dbs_obj()
FI_list = make_FI_list(dbs_obj)
log(20, 'xa_obj.get_FI_list: FI_list:{}, dbs_obj:{}'.format(FI_list, dbs_obj))
for dbName in FI_list:
log(30, '------xxxxxxXXXXXX NEW FI XXXXXXxxxxxx---------')
log(20, 'Current dbName:{}'.format(dbName))
bth_obj = xa_obj.get_bth_obj(dbName)
log(20, 'Get Batch Threshold Object(bth_obj) from DataBase for {}, bth_obj:{}'.format(dbName, bth_obj))
threshold_obj = ct.CurrentThreshold(width, bth_obj)
threshold_obj.process_bth_obj()
logger.info('google search successful {*args} {**kwargs}')
return urls
except HTTPError as e:
logger.info(f'{e} at backoff {backoff}')
return google_search(url, query, stop, backoff=backoff+1)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--newspapers', default=["all", ], nargs='*')
parser.add_argument('--num', default=10, nargs='?', type=int)
parser.add_argument('--source', default='google', nargs='?', type=str)
args = parser.parse_args()
logger = make_logger('logger.log')
logger.info(args)
url_source = args.source
# get newspaper info
newspapers = args.newspapers
if newspapers == ['all', ]:
newspapers = registry
else:
newspapers = [n for n in registry if n['newspaper_id'] in newspapers]
print(newspapers)
for newspaper in newspapers:
# collect_urls
parser = newspaper['parser']
checker = newspaper['checker']
from .caching.caching import get_cache
from .caching.global_cache import GlobalCache
from .caching.global_cache import ModelDataEnum
from .models import ClimateArea
from .models import Company
from .models import Device
from .models import Location
from .models import OutdoorRecord
from .models import OutdoorSpot
from .models import Project
from .models import ProjectDetail
from .models import Spot
from .models import SpotRecord
from .models import User
logger = make_logger('modelOperation', 'modelOperation_log', DEBUG)
logger.propagate = False
T = TypeVar('T')
# from app import global_cache
app = importlib.import_module('app')
global_cache = app.global_cache
# TODO lazy load global_cache.global_cacheall so it is fully initialized.
PostData = Dict
@global_cache.global_cacheall
def cacher_test(cache):
print(cache)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
try:
import logger
try:
import config
colored = getattr(config, 'log_colored', False)
except ImportError:
colored = False
logger = logger.make_logger('pserver', debug=False, colored=colored)
except ImportError:
import logging as logger
__author__ = "Robert Zaremba"
__version__ = version = "0.1"
__license__ = "Apache License v2"
from base import PServer
from protocols import *
def train(config_file, resume=False, **kwargs):
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
# [PersonReID_Dataset_Downloader(cfg.DATASETS.STORE_DIR,dataset) for dataset in cfg.DATASETS.SOURCE]
# [PersonReID_Dataset_Downloader(cfg.DATASETS.STORE_DIR,dataset) for dataset in cfg.DATASETS.TARGET]
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log', resume)
if not resume:
logger.info("Using {} GPUS".format(1))
logger.info("Loaded configuration file {}".format(config_file))
logger.info("Running with config:\n{}".format(cfg))
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
eval_period = cfg.SOLVER.EVAL_PERIOD
output_dir = cfg.OUTPUT_DIR
device = torch.device(cfg.DEVICE)
epochs = cfg.SOLVER.MAX_EPOCHS
train_loader, _, _, num_classes = data_loader(cfg,
cfg.DATASETS.SOURCE,
merge=cfg.DATASETS.MERGE)
model = getattr(models, cfg.MODEL.NAME)(num_classes, cfg.MODEL.LAST_STRIDE,
cfg.MODEL.POOL)
if resume:
checkpoints = get_last_stats(output_dir)
try:
model_dict = torch.load(checkpoints[cfg.MODEL.NAME])
except KeyError:
model_dict = torch.load(checkpoints[str(type(model))])
model.load_state_dict(model_dict)
if device:
model.to(device) # must be done before the optimizer generation
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
base_epo = 0
if resume:
optimizer.load_state_dict(torch.load(checkpoints['opt']))
sch_dict = torch.load(checkpoints['sch'])
scheduler.load_state_dict(sch_dict)
base_epo = checkpoints['epo']
loss_fn = make_loss(cfg)
if not resume:
logger.info("Start training")
since = time.time()
for epoch in range(epochs):
count = 0
running_loss = 0.0
running_acc = 0
for data in tqdm(train_loader, desc='Iteration', leave=False):
model.train()
images, labels, domains = data
if device:
model.to(device)
images, labels, domains = images.to(device), labels.to(
device), domains.to(device)
optimizer.zero_grad()
scores, feats = model(images)
loss = loss_fn(scores, feats, labels)
loss.backward()
optimizer.step()
count = count + 1
running_loss += loss.item()
running_acc += (
scores[0].max(1)[1] == labels).float().mean().item()
logger.info(
"Epoch[{}] Iteration[{}/{}] Loss: {:.3f}, Acc: {:.3f}, Base Lr: {:.2e}"
.format(epoch + 1 + base_epo, count, len(train_loader),
running_loss / count, running_acc / count,
scheduler.get_lr()[0]))
scheduler.step()
if (epoch + 1 + base_epo) % checkpoint_period == 0:
model.cpu()
model.save(output_dir, epoch + 1 + base_epo)
torch.save(
optimizer.state_dict(),
os.path.join(output_dir,
'opt_epo' + str(epoch + 1 + base_epo) + '.pth'))
torch.save(
scheduler.state_dict(),
os.path.join(output_dir,
'sch_epo' + str(epoch + 1 + base_epo) + '.pth'))
# Validation
if (epoch + base_epo + 1) % eval_period == 0:
# Validation on Target Dataset
for target in cfg.DATASETS.TARGET:
mAPs = []
cmcs = []
for i in range(iteration):
set_seeds(i)
_, val_loader, num_query, _ = data_loader(cfg, (target, ),
merge=False)
all_feats = []
all_pids = []
all_camids = []
since = time.time()
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
if device:
model.to(device)
images = images.to(device)
feats = model(images)
feats /= feats.norm(dim=-1, keepdim=True)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
cmc, mAP = evaluation(all_feats, all_pids, all_camids,
num_query)
mAPs.append(mAP)
cmcs.append(cmc)
mAP = np.mean(np.array(mAPs))
cmc = np.mean(np.array(cmcs), axis=0)
mAP_std = np.std(np.array(mAPs))
cmc_std = np.std(np.array(cmcs), axis=0)
logger.info("Validation Results: {} - Epoch: {}".format(
target, epoch + 1 + base_epo))
logger.info("mAP: {:.1%} (std: {:.3%})".format(mAP, mAP_std))
for r in [1, 5, 10]:
logger.info(
"CMC curve, Rank-{:<3}:{:.1%} (std: {:.3%})".format(
r, cmc[r - 1], cmc_std[r - 1]))
reset()
time_elapsed = time.time() - since
logger.info('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
logger.info('-' * 10)
import json
import sys
import os
import sql_queries as sql_q
from filters_json import filter_list
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from logger import make_logger
req_tok_url = 'https://api.twitter.com/oauth/request_token'
oauth_url = 'https://api.twitter.com/oauth/authorize'
acc_tok_url = 'https://api.twitter.com/oauth/access_token'
logger = make_logger('push_db', 'retoracle.log')
def return_filters():
filter_l = []
for i in filters:
for y in filters[i]['search_terms']:
for x in filters[i]['search_terms'][y]:
filter_l.append(x)
return filter_l
def return_blacklist():
blackList = []
for i in filters:
for y in filters[i]['blacklist']:
import time, datetime
import json
import sys
import os
import sql_queries as sql_q
from filters_json import filter_list
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from logger import make_logger
req_tok_url = 'https://api.twitter.com/oauth/request_token'
oauth_url = 'https://api.twitter.com/oauth/authorize'
acc_tok_url = 'https://api.twitter.com/oauth/access_token'
logger = make_logger('push_db', 'retoracle.log')
def return_filters():
filter_l = []
for i in filters:
for y in filters[i]['search_terms']:
for x in filters[i]['search_terms'][y]:
filter_l.append(x)
return filter_l
def return_blacklist():
blackList = []
for i in filters:
for y in filters[i]['blacklist']:
def train(config_file, **kwargs):
cfg.merge_from_file(config_file)
if kwargs:
opts = []
for k, v in kwargs.items():
opts.append(k)
opts.append(v)
cfg.merge_from_list(opts)
cfg.freeze()
#PersonReID_Dataset_Downloader('./datasets',cfg.DATASETS.NAMES)
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = make_logger("Reid_Baseline", output_dir, 'log')
logger.info("Using {} GPUS".format(1))
logger.info("Loaded configuration file {}".format(config_file))
logger.info("Running with config:\n{}".format(cfg))
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
eval_period = cfg.SOLVER.EVAL_PERIOD
output_dir = cfg.OUTPUT_DIR
device = torch.device(cfg.DEVICE)
epochs = cfg.SOLVER.MAX_EPOCHS
method = cfg.DATALOADER.SAMPLER
train_loader, val_loader, num_query, num_classes = data_loader(
cfg, cfg.DATASETS.NAMES)
model = getattr(models, cfg.MODEL.NAME)(num_classes, cfg.MODEL.LAST_STRIDE)
if 'center' in method:
loss_fn, center_criterion = make_loss(cfg)
optimizer, optimizer_center = make_optimizer_with_center(
cfg, model, center_criterion)
else:
loss_fn = make_loss(cfg)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
logger.info("Start training")
since = time.time()
for epoch in range(epochs):
count = 0
running_loss = 0.0
running_acc = 0
for data in tqdm(train_loader, desc='Iteration', leave=False):
model.train()
images, labels = data
if device:
model.to(device)
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
if 'center' in method:
optimizer_center.zero_grad()
scores, feats = model(images)
loss = loss_fn(scores, feats, labels)
loss.backward()
optimizer.step()
if 'center' in method:
for param in center_criterion.parameters():
param.grad.data *= (1. / cfg.SOLVER.CENTER_LOSS_WEIGHT)
optimizer_center.step()
count = count + 1
running_loss += loss.item()
running_acc += (scores.max(1)[1] == labels).float().mean().item()
logger.info(
"Epoch[{}] Iteration[{}/{}] Loss: {:.3f}, Acc: {:.3f}, Base Lr: {:.2e}"
.format(epoch + 1, count, len(train_loader), running_loss / count,
running_acc / count,
scheduler.get_lr()[0]))
scheduler.step()
if (epoch + 1) % checkpoint_period == 0:
model.cpu()
model.save(output_dir, epoch + 1)
# Validation
if (epoch + 1) % eval_period == 0:
all_feats = []
all_pids = []
all_camids = []
for data in tqdm(val_loader,
desc='Feature Extraction',
leave=False):
model.eval()
with torch.no_grad():
images, pids, camids = data
if device:
model.to(device)
images = images.to(device)
feats = model(images)
all_feats.append(feats)
all_pids.extend(np.asarray(pids))
all_camids.extend(np.asarray(camids))
logger.info("start evaluation")
cmc, mAP = evaluation(all_feats, all_pids, all_camids, num_query)
logger.info("Validation Results - Epoch: {}".format(epoch + 1))
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(
r, cmc[r - 1]))
time_elapsed = time.time() - since
logger.info('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
logger.info('-' * 10)
from pathlib import Path
# Environment Variables
env = os.environ
load_dotenv()
# The time in seconds the bot should sleep until it checks again.
SLEEP = 600
# LOGGING CONFIGURATION
LOG_FILENAME = "bot.log"
LOG_FILE_BACKUPCOUNT = 5
LOG_FILE_MAXSIZE = 1024 * 256
# LOGGING SETUP
log = logger.make_logger("bot", LOG_FILENAME, logging_level=logging.DEBUG)
# MAIN PROCEDURE
def run_bot():
# -- progmetalbot useragent and version --
app_useragent_version = env['APP_USERAGENT'] + ' ' + env[
'APP_VERSION'] + " by u/" + settings.USER_TO_MESSAGE
# -- praw --
reddit = praw.Reddit(user_agent=app_useragent_version,
client_id=env['REDDIT_CLIENT_ID'],
client_secret=env['REDDIT_CLIENT_SECRET'],
password=env['REDDIT_PASSWORD'],
username=env['REDDIT_USERNAME'])
subreddit = reddit.subreddit(settings.REDDIT_SUBREDDIT)
# -- musicbrainz --
