def download(self, name, path):
""" Download model denoted by the given name to disk.
:param name: Name of the model to download.
:param path: Path of the directory to save model into.
"""
url = '{}/{}/{}/{}/{}.tar.gz'.format(self._host, self._repository,
self.RELEASE_PATH, self._release,
name)
get_logger().info('Downloading model archive %s', url)
with requests.get(url, stream=True) as response:
response.raise_for_status()
archive = NamedTemporaryFile(delete=False)
try:
with archive as stream:
# Note: check for chunk size parameters ?
for chunk in response.iter_content(chunk_size=8192):
if chunk:
stream.write(chunk)
get_logger().info('Validating archive checksum')
if compute_file_checksum(archive.name) != self.checksum(name):
raise IOError('Downloaded file is corrupted, please retry')
get_logger().info('Extracting downloaded %s archive', name)
with tarfile.open(name=archive.name) as tar:
tar.extractall(path=path)
finally:
os.unlink(archive.name)
get_logger().info('%s model file(s) extracted', name)
def __init__(self, config):
super(StyleTextRec, self).__init__()
self.logger = get_logger()
self.text_generator = TextGenerator(config["Predictor"][
"text_generator"])
self.bg_generator = BgGeneratorWithMask(config["Predictor"][
"bg_generator"])
self.fusion_generator = FusionGeneratorSimple(config["Predictor"][
"fusion_generator"])
bg_generator_pretrain = config["Predictor"]["bg_generator"]["pretrain"]
text_generator_pretrain = config["Predictor"]["text_generator"][
"pretrain"]
fusion_generator_pretrain = config["Predictor"]["fusion_generator"][
"pretrain"]
load_dygraph_pretrain(
self.bg_generator,
self.logger,
path=bg_generator_pretrain,
load_static_weights=False)
load_dygraph_pretrain(
self.text_generator,
self.logger,
path=text_generator_pretrain,
load_static_weights=False)
load_dygraph_pretrain(
self.fusion_generator,
self.logger,
path=fusion_generator_pretrain,
load_static_weights=False)
def __init__(self, config):
self.logger = get_logger()
self.logger.info("using NumberCorpus")
self.num_list = "0123456789"
self.en_char_list = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
self.height = config["Global"]["image_height"]
self.max_width = config["Global"]["image_width"]
def __init__(self, config, tag):
self.logger = get_logger()
self.output_dir = config["Global"]["output_dir"]
self.counter = 0
self.label_dict = {}
self.tag = tag
self.label_file_index = 0
def __init__(self):
super(BaseAPIView).__init__()
self.req_data = None
self.start_time = None
self.end_time = None
self.duration = None
self.logger = get_logger(__name__).bind(
operation="{0}_view".format(self.operation))
def run(args):
log_dir = os.path.join(root_path, LOG_TOY_METRIC_PATH)
logger = get_logger(args.dataset, log_dir, __file__)
print = logger.info
makedirs(osp.join(root_path, RESULT_TOY_METRIC_PATH, 'a.py'))
for seed in range(1, 51):
_run(args, seed, print)
def __init__(self, config):
self.logger = get_logger()
self.max_width = config["Global"]["image_width"]
self.char_list = " 0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
self.height = config["Global"]["image_height"]
self.font_dict = {}
self.load_fonts(config["TextDrawer"]["fonts"])
self.support_languages = list(self.font_dict)
def run(args):
method = 'gp' if args.method == 'svgp' else args.method
log_dir = osp.join(LOG_REG_PATH, '%s_%s' % (args.dataset, method))
# setup logger
logger = get_logger(args.dataset, log_dir, __file__)
print = logger.info
# set jitter
if args.dataset in ['concrete']:
gfs.settings.set_jitter(3e-5)
if args.dataset in ['naval']:
gfs.settings.set_jitter(1e-4)
rmse_results, lld_results = [], []
test_rmse_results, test_lld_results = [], []
for seed in range(args.init_seed, args.n_runs + args.init_seed):
rmse, ll, test_rmse, test_lld = _run(args, seed, print)
rmse_results.append(rmse)
lld_results.append(ll)
test_rmse_results.append(test_rmse)
test_lld_results.append(test_lld)
print('test rmse = {}'.format(rmse_results))
print('test log likelihood = {}'.format(lld_results))
print("Test rmse = {}/{}".format(
np.mean(rmse_results),
np.std(rmse_results) / args.n_runs**0.5))
print("Test log likelihood = {}/{}".format(
np.mean(lld_results),
np.std(lld_results) / args.n_runs**0.5))
print('NOTE: Test result above output mean and std. errors')
result_dir = osp.join(RESULT_REG_PATH, '%s_%s' % (args.dataset, method))
with open(osp.join(result_dir, 'res.json'), 'w') as f:
res = {
'rmse':
[np.mean(rmse_results),
np.std(rmse_results) / args.n_runs**0.5],
'lld':
[np.mean(lld_results),
np.std(lld_results) / args.n_runs**0.5],
'test_rmse': [
np.mean(test_rmse_results),
np.std(test_rmse_results) / args.n_runs**0.5
],
'test_lld': [
np.mean(test_lld_results),
np.std(test_lld_results) / args.n_runs**0.5
],
'all_rmse':
rmse_results,
'all_lld':
lld_results
}
json.dump(res, f)
def run(args):
log_dir = osp.join(LOG_AL_PRETRAIN_ORACLE_PATH, args.dataset)
logger = get_logger(args.dataset, log_dir, __file__)
print = logger.info
# set jitter
if args.dataset in ['concrete']:
gfs.settings.set_jitter(3e-5)
if args.dataset in ['naval']:
gfs.settings.set_jitter(1e-4)
for seed in range(args.init_seed, args.n_runs + args.init_seed):
_run(args, seed, print)
def save(self, path, data, sample_rate, codec=None, bitrate=None):
""" Write waveform data to the file denoted by the given path
using FFMPEG process.
:param path: Path of the audio file to save data in.
:param data: Waveform data to write.
:param sample_rate: Sample rate to write file in.
:param codec: (Optional) Writing codec to use.
:param bitrate: (Optional) Bitrate of the written audio file.
:raise IOError: If any error occurs while using FFMPEG to write data.
"""
_check_ffmpeg_install()
directory = os.path.dirname(path)
if not os.path.exists(directory):
raise Exception(f'output directory does not exists: {directory}')
get_logger().debug('Writing file %s', path)
input_kwargs = {'ar': sample_rate, 'ac': data.shape[1]}
output_kwargs = {'ar': sample_rate, 'strict': '-2'}
print(data[0][0].astype('<f4').tobytes())
if bitrate:
output_kwargs['audio_bitrate'] = bitrate
if codec is not None and codec != 'wav':
output_kwargs['codec'] = _to_ffmpeg_codec(codec)
process = (ffmpeg.input(
'pipe:', format='f32le', **input_kwargs).output(
path,
**output_kwargs).overwrite_output().run_async(pipe_stdin=True,
pipe_stderr=True,
quiet=True))
try:
process.stdin.write(data.astype('<f4').tobytes())
process.stdin.close()
process.wait()
except IOError:
raise Exception(f'FFMPEG error: {process.stderr.read()}')
get_logger().info('File %s written succesfully', path)
def __init__(self, config):
algorithm = config['Predictor']['algorithm']
assert algorithm in ["StyleTextRec"
], "Generator {} not supported.".format(algorithm)
use_gpu = config["Global"]['use_gpu']
check_gpu(use_gpu)
self.logger = get_logger()
self.generator = getattr(style_text_rec, algorithm)(config)
self.height = config["Global"]["image_height"]
self.width = config["Global"]["image_width"]
self.scale = config["Predictor"]["scale"]
self.mean = config["Predictor"]["mean"]
self.std = config["Predictor"]["std"]
self.expand_result = config["Predictor"]["expand_result"]
def main(argv):
""" Spleeter runner. Parse provided command line arguments
and run entrypoint for required command (either train,
evaluate or separate).
:param argv: Provided command line arguments.
"""
try:
parser = create_argument_parser()
arguments = parser.parse_args(argv[1:])
enable_logging()
if arguments.verbose:
enable_tensorflow_logging()
if arguments.command == 'separate':
from commands.separate import entrypoint
elif arguments.command == 'train':
from commands.train import entrypoint
elif arguments.command == 'evaluate':
from commands.evaluate import entrypoint
params = load_configuration(arguments.configuration)
entrypoint(arguments, params)
except SpleeterError as e:
get_logger().error(e)
def __init__(self, config):
self.logger = get_logger()
self.logger.info("using FileCorpus")
self.char_list = " 0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
corpus_file = config["CorpusGenerator"]["corpus_file"]
self.language = config["CorpusGenerator"]["language"]
with open(corpus_file, 'r') as f:
corpus_raw = f.read()
self.corpus_list = corpus_raw.split("\n")[:-1]
assert len(self.corpus_list) > 0
random.shuffle(self.corpus_list)
self.index = 0
def __init__(self):
self.FLAGS = ArgsParser().parse_args()
self.config = load_config(self.FLAGS.config)
self.config = override_config(self.config, options=self.FLAGS.override)
self.output_dir = self.config["Global"]["output_dir"]
if not os.path.exists(self.output_dir):
os.mkdir(self.output_dir)
self.logger = get_logger(
log_file='{}/predict.log'.format(self.output_dir))
self.text_drawer = text_drawers.StdTextDrawer(self.config)
predictor_method = self.config["Predictor"]["method"]
assert predictor_method is not None
self.predictor = getattr(predictors, predictor_method)(self.config)
def safe_load(path, offset, duration, sample_rate, dtype):
logger = get_logger()
logger.info(
f'Loading audio {path} from {offset} to {offset + duration}')
try:
(data, _) = self.load(path.numpy(),
offset.numpy(),
duration.numpy(),
sample_rate.numpy(),
dtype=dtype.numpy())
logger.info('Audio data loaded successfully')
return (data, False)
except Exception as e:
logger.exception('An error occurs while loading audio',
exc_info=e)
return (np.float32(-1.0), True)
def run(args):
method = 'gp' if args.method == 'svgp' else args.method
base_method = 'gp' if args.base_method == 'svgp' else args.base_method
log_dir = osp.join(LOG_AL_PREDICTION_PATH, '%s_%s_%s'%(args.dataset, method, base_method))
logger = get_logger(args.dataset, log_dir, __file__)
print = logger.info
# set jitter
if args.dataset in ['concrete']:
gfs.settings.set_jitter(3e-5)
if args.dataset in ['naval']:
gfs.settings.set_jitter(1e-4)
rmse_results, lld_results = [], []
for seed in range(args.init_seed, args.n_runs+args.init_seed):
rmse, ll = _run(args, seed, print)
rmse_results.append(rmse)
lld_results.append(ll)
rmse_results = np.array(rmse_results)
lld_results = np.array(lld_results)
rmse_mean = np.mean(rmse_results, axis=0)
rmse_var = np.var(rmse_results, axis=0)
lld_mean = np.mean(lld_results, axis=0)
lld_var = np.var(lld_results, axis=0)
print("rmse_mean: %s" % rmse_mean.tolist())
print("rmse_var: %s" % rmse_var.tolist())
print("lld_mean: %s" % lld_mean.tolist())
print("lld_var: %s" % lld_var.tolist())
result_dir = osp.join(RESULT_AL_PATH, '%s_%s_%s.json'%(args.dataset, method, base_method))
with open(result_dir, 'w') as f:
res = {
'dataset': args.dataset,
'criteria': args.criteria,
'rmse_mean': rmse_mean.tolist(),
'rmse_var': rmse_var.tolist(),
'lld_mean': lld_mean.tolist(),
'lld_var': lld_var.tolist(),
'rmse': rmse_results.tolist(),
'lld': lld_results.tolist(),
}
json.dump(res, f)
def main(args):
# Setting the random seed throughout the modules
if args.seed is not None:
np.random.seed(args.seed)
th.manual_seed(args.seed)
args.env_args['seed'] = args.seed
config = OmegaConf.to_container(args)
if args.use_wandb:
wandb.login()
wandb.init(entity=args.wandb_entity,
project=args.wandb_project,
group=args.wandb_group,
config=config)
# run the framework
logger = get_logger()
run(config, logger)
import os
import collections
from os.path import dirname, abspath
from copy import deepcopy
from sacred import Experiment, SETTINGS
from sacred.observers import FileStorageObserver
from sacred.utils import apply_backspaces_and_linefeeds
import sys
import torch as th
from utils.logging import get_logger
import yaml
from run import run
SETTINGS['CAPTURE_MODE'] = "fd" # set to "no" if you want to see stdout/stderr in console
logger = get_logger()
ex = Experiment("pymarl")
ex.logger = logger
ex.captured_out_filter = apply_backspaces_and_linefeeds
results_path = os.path.join(dirname(dirname(abspath(__file__))), "results")
@ex.main
def my_main(_run, _config, _log):
# Setting the random seed throughout the modules
config = config_copy(_config)
np.random.seed(config["seed"])
th.manual_seed(config["seed"])
config['env_args']['seed'] = config["seed"]
from __future__ import absolute_import, unicode_literals
import logging
import os
import sys
from utils.logging import get_logger
CORE_MODULES_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'core_modules')
MODULES_PATHS = [CORE_MODULES_PATH]
logger = get_logger('module', logging.DEBUG)
class BaseModule(object):
"""
Base class for creating a module.
Attributes:
id Unique identifier for this module
title Title of the app
description Description of the app
dependencies List of other module needed for this one to work.
handlers List of irc handlers
"""
identifier = ''
title = ''
description = ''
dependencies = []
handlers = []
def main_worker(local_rank: int, config: object):
torch.cuda.set_device(local_rank)
if config.distributed:
raise NotImplementedError
config.batch_size = config.batch_size // config.num_gpus_per_node
config.num_workers = max(1, config.num_workers // config.num_gpus_per_node)
in_channels = int(config.decouple_input) + 1
# Model
BACKBONE_CONFIGS, Backbone = AVAILABLE_MODELS[config.backbone_type]
Projector = PROJECTOR_TYPES[config.projector_type]
encoder = Backbone(BACKBONE_CONFIGS[config.backbone_config],
in_channels=in_channels)
head = Projector(encoder.out_channels, config.projector_size)
# Optimization
params = [{'params': encoder.parameters()}, {'params': head.parameters()}]
optimizer = get_optimizer(params=params,
name=config.optimizer,
lr=config.learning_rate,
weight_decay=config.weight_decay,
momentum=config.momentum)
scheduler = get_scheduler(optimizer=optimizer,
name=config.scheduler,
epochs=config.epochs,
warmup_steps=config.warmup_steps)
# Data
data_kwargs = {
'transform':
WM811KTransform(size=config.input_size, mode='test'),
'positive_transform':
WM811KTransform(size=config.input_size, mode=config.augmentation),
'decouple_input':
config.decouple_input,
}
train_set = torch.utils.data.ConcatDataset([
WM811KForWaPIRL('./data/wm811k/unlabeled/train/', **data_kwargs),
WM811KForWaPIRL('./data/wm811k/labeled/train/', **data_kwargs),
])
valid_set = torch.utils.data.ConcatDataset([
WM811KForWaPIRL('./data/wm811k/unlabeled/valid/', **data_kwargs),
WM811KForWaPIRL('./data/wm811k/labeled/valid/', **data_kwargs),
])
test_set = torch.utils.data.ConcatDataset([
WM811KForWaPIRL('./data/wm811k/unlabeled/test/', **data_kwargs),
WM811KForWaPIRL('./data/wm811k/labeled/test/', **data_kwargs),
])
# Experiment (WaPIRL)
experiment_kwargs = {
'backbone':
encoder,
'projector':
head,
'memory':
MemoryBank(size=(len(train_set), config.projector_size),
device=local_rank),
'optimizer':
optimizer,
'scheduler':
scheduler,
'loss_function':
WaPIRLLoss(temperature=config.temperature),
'loss_weight':
config.loss_weight,
'num_negatives':
config.num_negatives,
'distributed':
config.distributed,
'local_rank':
local_rank,
'metrics': {
'top@1': TopKAccuracy(num_classes=1 + config.num_negatives, k=1),
'top@5': TopKAccuracy(num_classes=1 + config.num_negatives, k=5)
},
'checkpoint_dir':
config.checkpoint_dir,
'write_summary':
config.write_summary,
}
experiment = WaPIRL(**experiment_kwargs)
if local_rank == 0:
logfile = os.path.join(config.checkpoint_dir, 'main.log')
logger = get_logger(stream=False, logfile=logfile)
logger.info(f"Data: {config.data}")
logger.info(f"Augmentation: {config.augmentation}")
logger.info(f"Observations: {len(train_set):,}")
logger.info(
f"Trainable parameters ({encoder.__class__.__name__}): {encoder.num_parameters:,}"
)
logger.info(
f"Trainable parameters ({head.__class__.__name__}): {head.num_parameters:,}"
)
logger.info(
f"Projection head: {config.projector_type} ({config.projector_size})"
)
logger.info(f"Checkpoint directory: {config.checkpoint_dir}")
else:
logger = None
# Train (WaPIRL)
run_kwargs = {
'train_set': train_set,
'valid_set': valid_set,
'epochs': config.epochs,
'batch_size': config.batch_size,
'num_workers': config.num_workers,
'logger': logger,
'save_every': config.save_every,
}
experiment.run(**run_kwargs)
if logger is not None:
logger.handlers.clear()
parser.add_argument('-r', '--rand', type=str, default='uniform')
parser.add_argument('-na', '--n_rand', type=int, default=5)
parser.add_argument('-nh', '--n_hidden', type=int, default=1)
parser.add_argument('-nu', '--n_units', type=int, default=50)
parser.add_argument('-bs', '--batch_size', type=int, default=20)
parser.add_argument('-lr', '--learning_rate', type=float, default=0.001)
parser.add_argument('-e', '--epochs', type=int, default=2000)
parser.add_argument('--n_eigen_threshold', type=float, default=0.99)
parser.add_argument('--train_samples', type=int, default=100)
parser.add_argument('--test_samples', type=int, default=100)
parser.add_argument('--print_interval', type=int, default=100)
parser.add_argument('--test_interval', type=int, default=100)
args = parser.parse_args()
logger = get_logger(args.dataset, 'results/regression/%s/' % args.dataset,
__file__)
print = logger.info
def run(seed):
tf.reset_default_graph()
############################## load and normalize data ##############################
dataset = uci_woval(args.dataset, seed=seed)
train_x, test_x, train_y, test_y = dataset.x_train, dataset.x_test, dataset.y_train, dataset.y_test
std_y_train = dataset.std_y_train[0]
N, input_dim = train_x.shape
lower_ap = np.minimum(np.min(train_x), np.min(test_x))
upper_ap = np.maximum(np.max(train_x), np.max(test_x))
mean_x_train, std_x_train = np.mean(train_x, 0), np.std(train_x, 0)
from logging import DEBUG
import socket
from threading import Thread
import schedule
import six
import time
from irc.handlers import handle_connexion, handle_commands
from irc.parser import Response, InvalidResponse
from irc.sender import Sender
from utils.config import Configuration
from utils.logging import get_logger
logger = get_logger('irc', DEBUG)
HOST = 'irc.twitch.tv'
#HOST = 'localhost'
PORT = 6667
class Client(object):
""" Connect to IRC and dispatch the messages to the handlers. """
def __init__(self, pipe):
self.pipe = pipe
self.config = Configuration.get()
self.socket = None
self.handlers = []
self.add_handler(handle_commands)
from watchdog.events import FileSystemEventHandler
import paho.mqtt.client as mqtt
import json
import requests
from utils import logging
from threading import Thread
from threading import Timer
from probe import probe, run
import time
import os
import tempfile
import shutil
import re
logger = logging.get_logger('main', is_static=True)
class Feeder():
def __init__(self):
logger.debug("Initializing Feeder")
self.office = list(map(float, os.environ["OFFICE"].split(",")))
self.alg_id = None
self.recording_volume = os.environ["STORAGE_VOLUME"]
self.every_nth_frame = int(os.environ["EVERY_NTH_FRAME"])
#Hosts
self.dbhost = os.environ["DBHOST"]
self.vahost = "http://localhost:8080/pipelines"
self.mqtthost = os.environ["MQTTHOST"]
SEED = 1
DATA_PATH = '/project/cq-training-1/project1/data/'
HDF5_8BIT = 'hdf5v7_8bit'
BATCH_LOG_INTERVAL = 50
VALID_PERC = 0.2
SLURM_TMPDIR = os.environ["SLURM_TMPDIR"] if "SLURM_TMPDIR" in os.environ else \
glob.glob('/localscratch/'+os.environ['USER']+'*')[0]
# Setup writers for tensorboard
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/gradient_tape/' + current_time + '/train'
test_log_dir = 'logs/gradient_tape/' + current_time + '/valid'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir)
logger = logging.get_logger()
# Metrics
train_mse_metric = tf.keras.metrics.MeanSquaredError()
valid_mse_metric = tf.keras.metrics.MeanSquaredError()
valid_csky_mse_metric = tf.keras.metrics.MeanSquaredError()
def train_epoch(model, data_loader, batch_size, loss_function, optimizer,
total_examples, scale_label, use_csky):
train_mse_metric.reset_states()
for i, batch in tqdm(enumerate(data_loader),
total=(np.ceil(total_examples / batch_size)),
desc='train epoch',
leave=False):
images, labels, csky = batch['images'], batch['ghi'], batch['csky_ghi']
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
"""
A script to benchmark data loading.
"""
import utils.logging as logging
from utils.benchmark import benchmark_data_loading
from utils.misc import launch_job
from utils.parser import load_config, parse_args
logger = logging.get_logger(__name__)
def main():
args = parse_args()
cfg = load_config(args)
launch_job(cfg=cfg,
init_method=args.init_method,
func=benchmark_data_loading)
if __name__ == "__main__":
main()
import torch
import utils.logging as logging
from utils import lr_policy
logger = logging.get_logger('default')
def get_optim_params(model, cfg):
normal_params = []
feature_extract_params = []
deformable_prediction_params = []
for k, v in model.named_parameters():
if v.requires_grad:
if ('feature_extraction' in k) or ('fea_L2' in k) or ('fea_L3' in k):
feature_extract_params.append(v)
elif 'conv_offset_mask' in k:
deformable_prediction_params.append(v)
else:
normal_params.append(v)
# else:
# if self.rank <= 0:
# logger.warning('Params [{:s}] will not optimize.'.format(k))
optim_params = [
{ # add normal params first
'params': normal_params,
'lr': cfg.SOLVER.BASE_LR
},
{
'params': feature_extract_params,
'lr': cfg.SOLVER.BASE_LR
},
{
parser.add_argument('-il', '--init_logstd', type=float, default=-5.)
parser.add_argument('-na', '--n_rand', type=int, default=100)
parser.add_argument('-nh', '--n_hidden', type=int, default=2)
parser.add_argument('-nu', '--n_units', type=int, default=100)
parser.add_argument('-lr', '--learning_rate', type=float, default=0.003)
parser.add_argument('-e', '--epochs', type=int, default=30000)
parser.add_argument('--n_eigen_threshold', type=float, default=0.9)
parser.add_argument('--train_samples', type=int, default=500)
parser.add_argument('--test_samples', type=int, default=100)
parser.add_argument('--print_interval', type=int, default=100)
parser.add_argument('--test_interval', type=int, default=2000)
parser.add_argument('--seed', type=int, default=8)
args = parser.parse_args()
logger = get_logger(args.dataset, 'results/%s/' % (args.dataset), __file__)
print = logger.info
tf.set_random_seed(args.seed)
np.random.seed(args.seed)
xmin, xmax = 0., 1.
ymin, ymax = 0., 1.
lambda_, y_std = 3., 0.02
############################## setup FBNN model ##############################
prior_generator = dict(p_lin=PiecewiseLinear,
p_const=PiecewiseConstant)[args.dataset](lambda_, xmin,
xmax, ymin,
ymax).sample
def rand_generator(*arg):
import sys
from functools import wraps
from time import perf_counter
from utils.logging import get_logger
logger = get_logger(__name__)
def timer(f):
@wraps(f)
def wrapper(*args, **kwargs):
start = perf_counter()
resp = f(*args, **kwargs)
end = perf_counter()
logger.info('{}, {}'.format(f.__name__, end - start))
return resp
return wrapper
_intervals = (
('weeks', 604800, int), # 60 * 60 * 24 * 7
('days', 86400, int), # 60 * 60 * 24
('hours', 3600, int), # 60 * 60
('minutes', 60, int),
('seconds', 1.0, float),
)
def display_time(seconds):
result = []
from __future__ import absolute_import, unicode_literals
from logging import DEBUG
import time
import six
from utils.logging import get_logger
logger = get_logger('irc_sender', DEBUG)
# Max message per window
MAX_MESSAGE = 20
# Moderators can send 100 message per 30 seconds
MAX_MESSAGE_MOD = 100
# In seconds
WINDOW_SIZE = 30
class Sender(object):
def __init__(self, channel):
self.channel = channel
self.buffer = []
self.sent = {}
# Handlers which generate the raw messages
def raw(self, message, **kwargs):
return message
def who(self, message, **kwargs):
import pika
from utils import logging
from utils.config import props
LOGGER = logging.get_logger()
RMQ_PORT = props["RABBIT_MQ"]["RMQ_PORT"]
RMQ_HOST = props["RABBIT_MQ"]["RMQ_HOST"]
RMQ_HEARTBEAT_INTERVAL = props.getint("RABBIT_MQ", "RMQ_HEARTBEAT_INTERVAL")
RMQ_INPUT_QUEUE = props["RABBIT_MQ"]["RMQ_INPUT_QUEUE_NAME"]
RMQ_OUTPUT_QUEUE = props["RABBIT_MQ"]["RMQ_OUTPUT_QUEUE_NAME"]
RMQ_EXCHANGE_QUEUE = props["RABBIT_MQ"]["RMQ_EXCHANGE_NAME"]
RMQ_USER = props.get("RABBIT_MQ", "RMQ_USERNAME")
RMQ_PASSWORD = props.get("RABBIT_MQ", "RMQ_PASSWORD")
def init_rabbitmq_connection():
connection = pika.BlockingConnection(
pika.ConnectionParameters(host="localhost", port=5672))
channel = connection.channel()
channel.queue_declare(queue=RMQ_INPUT_QUEUE)
return connection, channel
logger.info("---- MAYA ----")
logger.info("Command: {}".format(" ".join(maya_command)))
subprocess.check_call(maya_command, cwd=ROOT_DIR)
# remove temporary files
if not keep_temp:
shutil.rmtree(temp_dir)
# ----------------------------------------------------------------------------
if __name__ == '__main__':
# convert data
config = flags.FLAGS
config(sys.argv)
# get logger
logger = logging.get_logger(config.output_dir)
# convert
video2mocap(
config.video_path,
config.output_dir,
config.keep_temp,
config.mayapy_exe,
config.python2_exe,
config.python3_exe,
)
def main(args):
"""Main function."""
# 1. Configurations
torch.backends.cudnn.benchmark = True
ENCODER_CONFIGS, DECODER_CONFIGS, Config, Encoder, Decoder = \
AVAILABLE_MODELS[args.backbone_type]
config = Config(args)
config.save()
logfile = os.path.join(config.checkpoint_dir, 'main.log')
logger = get_logger(stream=False, logfile=logfile)
# 2. Data
input_transform = get_transform(config.data,
size=config.input_size,
mode=config.augmentation,
noise=config.noise)
target_transform = get_transform(config.data,
size=config.input_size,
mode='test')
if config.data == 'wm811k':
train_set = torch.utils.data.ConcatDataset([
WM811KForDenoising('./data/wm811k/unlabeled/train/',
input_transform, target_transform),
WM811KForDenoising('./data/wm811k/labeled/train/', input_transform,
target_transform),
])
valid_set = torch.utils.data.ConcatDataset([
WM811KForDenoising('./data/wm811k/unlabeled/valid/',
input_transform, target_transform),
WM811KForDenoising('./data/wm811k/labeled/valid/', input_transform,
target_transform),
])
test_set = torch.utils.data.ConcatDataset([
WM811KForDenoising('./data/wm811k/unlabeled/test/',
input_transform, target_transform),
WM811KForDenoising('./data/wm811k/labeled/test/', input_transform,
target_transform),
])
else:
raise ValueError(
f"Denoising only supports 'wm811k' data. Received '{config.data}'."
)
# 3. Model
encoder = Encoder(RESNET_ENCODER_CONFIGS[config.backbone_config],
in_channels=IN_CHANNELS[config.data])
decoder = Decoder(RESNET_DECODER_CONFIGS[config.backbone_config],
input_shape=encoder.output_shape,
output_shape=(OUT_CHANNELS[config.data],
config.input_size, config.input_size))
# 4. Optimization
params = [{
'params': encoder.parameters()
}, {
'params': decoder.parameters()
}]
optimizer = get_optimizer(params=params,
name=config.optimizer,
lr=config.learning_rate,
weight_decay=config.weight_decay,
momentum=config.momentum)
scheduler = get_scheduler(optimizer=optimizer,
name=config.scheduler,
epochs=config.epochs,
milestone=config.milestone,
warmup_steps=config.warmup_steps)
# 5. Experiment (Denoising)
experiment_kwargs = {
'encoder': encoder,
'decoder': decoder,
'optimizer': optimizer,
'scheduler': scheduler,
'loss_function': nn.CrossEntropyLoss(reduction='mean'),
'metrics': None,
'checkpoint_dir': config.checkpoint_dir,
'write_summary': config.write_summary,
}
experiment = Denoising(**experiment_kwargs)
# 6. Run (train, evaluate, and test model)
run_kwargs = {
'train_set': train_set,
'valid_set': valid_set,
'test_set': test_set,
'epochs': config.epochs,
'batch_size': config.batch_size,
'num_workers': config.num_workers,
'device': config.device,
'logger': logger,
'save_every': config.save_every,
}
logger.info(f"Data: {config.data}")
logger.info(f"Augmentation: {config.augmentation}")
logger.info(
f"Train : Valid : Test = {len(train_set):,} : {len(valid_set):,} : {len(test_set):,}"
)
logger.info(
f"Trainable parameters ({encoder.__class__.__name__}): {encoder.num_parameters:,}"
)
logger.info(
f"Trainable parameters ({decoder.__class__.__name__}): {decoder.num_parameters:,}"
)
logger.info(f"Saving model checkpoints to: {experiment.checkpoint_dir}")
logger.info(
f"Epochs: {run_kwargs['epochs']}, Batch size: {run_kwargs['batch_size']}"
)
logger.info(
f"Workers: {run_kwargs['num_workers']}, Device: {run_kwargs['device']}"
)
steps_per_epoch = len(train_set) // config.batch_size + 1
logger.info(f"Training steps per epoch: {steps_per_epoch:,}")
logger.info(
f"Total number of training iterations: {steps_per_epoch * config.epochs:,}"
)
experiment.run(**run_kwargs)
logger.handlers.clear()
def main(args):
"""Main function."""
# 1. Configurations
torch.backends.cudnn.benchmark = True
BACKBONE_CONFIGS, Config, Backbone = AVAILABLE_MODELS[args.backbone_type]
Projector = PROJECTOR_TYPES[args.projector_type]
config = Config(args)
config.save()
logfile = os.path.join(config.checkpoint_dir, 'main.log')
logger = get_logger(stream=False, logfile=logfile)
# 2. Data
if config.data == 'wm811k':
data_transforms = {
'transform':
get_transform(data=config.data,
size=config.input_size,
mode='test'),
'positive_transform':
get_transform(
data=config.data,
size=config.input_size,
mode=config.augmentation,
),
}
train_set = torch.utils.data.ConcatDataset([
WM811KForPIRL('./data/wm811k/unlabeled/train/', **data_transforms),
WM811KForPIRL('./data/wm811k/labeled/train/', **data_transforms),
])
valid_set = torch.utils.data.ConcatDataset([
WM811KForPIRL('./data/wm811k/unlabeled/valid/', **data_transforms),
WM811KForPIRL('./data/wm811k/labeled/valid/', **data_transforms),
])
test_set = torch.utils.data.ConcatDataset([
WM811KForPIRL('./data/wm811k/unlabeled/test/', **data_transforms),
WM811KForPIRL('./data/wm811k/labeled/test/', **data_transforms),
])
else:
raise ValueError(
f"PIRL only supports 'wm811k' data. Received '{config.data}'.")
# 3. Model
backbone = Backbone(BACKBONE_CONFIGS[config.backbone_config],
in_channels=IN_CHANNELS[config.data])
projector = Projector(backbone.out_channels, config.projector_size)
# 4. Optimization
params = [{
'params': backbone.parameters()
}, {
'params': projector.parameters()
}]
optimizer = get_optimizer(params=params,
name=config.optimizer,
lr=config.learning_rate,
weight_decay=config.weight_decay,
momentum=config.momentum)
scheduler = get_scheduler(optimizer=optimizer,
name=config.scheduler,
epochs=config.epochs,
milestone=config.milestone,
warmup_steps=config.warmup_steps)
# 5. Experiment (PIRL)
experiment_kwargs = {
'backbone':
backbone,
'projector':
projector,
'memory':
MemoryBank(size=(len(train_set), config.projector_size),
device=config.device),
'optimizer':
optimizer,
'scheduler':
scheduler,
'loss_function':
PIRLLoss(temperature=config.temperature),
'loss_weight':
config.loss_weight,
'num_negatives':
config.num_negatives,
'metrics': {
'top@1': TopKAccuracy(num_classes=1 + config.num_negatives, k=1),
'top@5': TopKAccuracy(num_classes=1 + config.num_negatives, k=5)
},
'checkpoint_dir':
config.checkpoint_dir,
'write_summary':
config.write_summary,
}
experiment = PIRL(**experiment_kwargs)
# 6. Run (train, evaluate, and test model)
run_kwargs = {
'train_set': train_set,
'valid_set': valid_set,
'test_set': test_set,
'epochs': config.epochs,
'batch_size': config.batch_size,
'num_workers': config.num_workers,
'device': config.device,
'logger': logger,
'save_every': config.save_every,
}
logger.info(f"Data: {config.data}")
logger.info(f"Augmentation: {config.augmentation}")
logger.info(
f"Train : Valid : Test = {len(train_set):,} : {len(valid_set):,} : {len(test_set):,}"
)
logger.info(
f"Trainable parameters ({backbone.__class__.__name__}): {backbone.num_parameters:,}"
)
logger.info(
f"Trainable parameters ({projector.__class__.__name__}): {projector.num_parameters:,}"
)
logger.info(f"Projector type: {config.projector_type}")
logger.info(f"Projector dimension: {config.projector_size}")
logger.info(f"Saving model checkpoints to: {experiment.checkpoint_dir}")
logger.info(
f"Epochs: {run_kwargs['epochs']}, Batch size: {run_kwargs['batch_size']}"
)
logger.info(
f"Workers: {run_kwargs['num_workers']}, Device: {run_kwargs['device']}"
)
steps_per_epoch = len(train_set) // config.batch_size + 1
logger.info(f"Training steps per epoch: {steps_per_epoch:,}")
logger.info(
f"Total number of training iterations: {steps_per_epoch * config.epochs:,}"
)
if config.resume_from_checkpoint is not None:
logger.info(
f"Resuming from a checkpoint: {config.resume_from_checkpoint}")
model_ckpt = os.path.join(config.resume_from_checkpoint,
'best_model.pt')
memory_ckpt = os.path.join(config.resume_from_checkpoint,
'best_memory.pt')
experiment.load_model_from_checkpoint(
model_ckpt) # load model & optimizer
experiment.memory.load(memory_ckpt) # load memory bank
# Assign optimizer variables to appropriate device
for state in experiment.optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(config.device)
experiment.run(**run_kwargs)
logger.handlers.clear()
