def main():
experiment_name = "Resnet_DY_05"
epoch = 100
experiment_path = os.path.join("experiments", experiment_name)
opt = Options(suppress_parse=True)
opt.load_from_file(os.path.join(experiment_path, "config.yaml"))
opt.checkpoint_path = os.path.join(experiment_path,
f"{experiment_name}_{epoch}.pth")
assert opt.use_dynamic
model = get_inspect_model(opt)
test_dl = data.create_data_loader(opt, "test")
test_score = test(model, opt.temperature[1], test_dl, opt.device)
entropy = compute_entropy(attentions_register)
print(test_score)
print(entropy)
def start():
# very good Mean Square Error: 2.433159948965294e-11
# options = Options(max_iter=1000, learning_rate=0.1, max_polynomial_degree=8,
# divisions_quantity=7, train_percents=0.8)
options = Options(max_iter=1000, learning_rate=0.1, max_polynomial_degree=8,
divisions_quantity=7, train_percents=0.75, alpha_iterations=13 ** 5, alpha_learning_rate=0.01)
files_handler = FilesHandler()
set_points = files_handler.get_points_from_file(files_handler.get_set_file_from_options())
test_points = files_handler.get_test_points_from_stdin()
apply_calculations(options, set_points, test_points)
return 0
def parse_args(argv):
"""
Parses command line arguments form an options object
:param argv:
:return:
"""
parser = ArgumentParser(prog="stock-correlated-news-harvester")
parser.add_argument('--stock_histories_file_path', metavar='Stock File Path',
type=str, required=True)
parser.add_argument('--stock_symbol_mapping_file_path', metavar='Stock Symbol Mapping File Path',
type=str, required=True)
parser.add_argument('--twitter_config_file_path', metavar='Twitter Credentials Config File',
type=str, required=True)
parser.add_argument('--results_path', metavar='Results Path',
type=str, required=True)
return parser.parse_args(argv, namespace=Options())
def main():
fpath = argv[1]
with open(fpath, 'r') as f:
data = f.read()
auto_process_libs = [
'math',
'os.path',
]
auto_import_libs = [
*auto_process_libs,
'os',
]
options = Options(
debug=True,
imports=auto_import_libs,
eval_mod=auto_process_libs,
)
data = PreProcessor(options=options).preprocess(data)
data = Transpiler(options=options).transpile(data)
tree = parse(data)
tree = Generator(options=options).generate(tree)
tree = Optimizer(options=options).optimize(tree)
tree = Inliner(options=options).inline(tree)
tree = Importer( options=options).clean_imports(tree)
tree = UnusedRemover(options=options).remove_unused(tree)
code = Unparser.unparse(tree)
code = Minifier(options=options).minify(code)
log(dump(tree))
log(code.replace('\n', '\\n\n'))
path_parts = fpath.split('.')
out_path = join(getcwd(), 'dist',
''.join(path_parts[:-1])[1:] + '.py')
out_dir = dirname(out_path)
if not exists(out_dir):
mkdir(out_dir)
with open(out_path, 'w') as f:
f.write(code)
import numpy as np
import os
import torch
import torch.nn as nn
import torch.multiprocessing as mp
from utils.options import Options
from utils.factory import GlobalLogsDict, ActorLogsDict, LearnerLogsDict, EvaluatorLogsDict
from utils.factory import LoggersDict, ActorsDict, LearnersDict, EvaluatorsDict, TestersDict
from utils.factory import EnvsDict, MemoriesDict, ModelsDict
if __name__ == '__main__':
mp.set_start_method("spawn", force=True)
opt = Options()
torch.manual_seed(opt.seed)
env_prototype = EnvsDict[opt.env_type]
memory_prototype = MemoriesDict[opt.memory_type]
model_prototype = ModelsDict[opt.model_type]
# dummy env to get state/action/reward/gamma/terminal_shape & action_space
dummy_env = env_prototype(opt.env_params, 0)
opt.state_shape = dummy_env.state_shape
opt.action_shape = dummy_env.action_shape
opt.action_space = dummy_env.action_space
opt.reward_shape = opt.agent_params.num_tasks
opt.gamma_shape = opt.agent_params.num_tasks
opt.terminal_shape = opt.agent_params.num_tasks
del dummy_env
from data.robotic_contour_dataloader import get_train_dataloader, get_val_dataloader
from utils.options import Options
from data.utils.prepare_data import get_split
from train_engine_contour import TrainEngine
if __name__ == '__main__':
opt = Options().opt
train_files, test_files = get_split(opt.fold)
train_dataloader = get_train_dataloader(train_files, opt)
val_dataloader = get_val_dataloader(test_files, opt)
engine = TrainEngine(opt)
engine.set_data(train_dataloader, val_dataloader)
engine.train_model()
num_workers=opt.workers,
pin_memory=True)
return train_dataloader
def get_val_dataloader(file_list, opt):
data_transform = Compose([
PadIfNeeded(
min_height=opt.val_crop_height, min_width=opt.val_crop_width, p=1),
CenterCrop(height=opt.val_crop_height, width=opt.val_crop_width, p=1),
Normalize(p=1)
],
p=1)
val_dataset = RoboticsDataset(file_names=file_list,
transform=data_transform,
problem_type=opt.problem_type)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.workers,
pin_memory=True)
return val_dataloader
if __name__ == '__main__':
# Test code for dataloader
options = Options()
train_files, test_files = prepare_data.get_split(0)
get_train_dataloader(train_files, options.opt)
from torchvision import transforms
from dataset.dataloader import ISICKerasDataset
from model.pix2pix import Pix2Pix
from torch.utils.data import DataLoader
transformations = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ToTensor()
])
eval_transformations = transforms.Compose([transforms.ToTensor()])
if __name__ == "__main__":
set_logger('train.log')
# prepare params for training
opt = Options()
args = opt.opts()
args = opt.params(args)
# Create datasetloader
train_dataset = ISICKerasDataset(args.dataset_dir,
data_type='train',
transform=transformations)
train_datasetLoder = DataLoader(train_dataset,
shuffle=True,
batch_size=args.batch_size,
num_workers=0)
# train_img = train_dataset[0]
eval_dataset = ISICKerasDataset(args.dataset_dir,
#!/usr/bin/env python3 # ------------------------------------------------------------------------------ # # Author: # Armin Hasitzka ([email protected]) # # Licensed under the MIT license. # See LICENSE in the project root for license information. # # ------------------------------------------------------------------------------ from enhancitizer import Enhancitizer from utils.options import Options if __name__ == '__main__': Enhancitizer(Options().collect()).run()
import os
import time
import tensorflow as tf
from utils.options import Options
from data.dataset import Dataset
from models.cyclegan import CycleGANModel
tf.enable_eager_execution()
"""
Run this module for training.
Required args: --data_dir, --save_dir
"""
if __name__ == "__main__":
opt = Options().parse(training=True)
dataset = Dataset(opt)
model = CycleGANModel(opt)
device = ("/gpu:" + str(opt.gpu_id)) if opt.gpu_id != -1 else "/cpu:0"
with tf.device(device):
global_step = model.global_step
batches_per_epoch = dataset.get_batches_per_epoch(opt)
# Initialize Tensorboard summary writer:
log_dir = os.path.join(opt.save_dir, 'tensorboard')
summary_writer = tf.contrib.summary.create_file_writer(log_dir, flush_millis=10000)
for epoch in range(1, opt.epochs):
start = time.time()
for train_step in range(batches_per_epoch):
# Record summaries every 100 train_steps, we multiply by 3 because there are 3 gradient updates per step.
# The Controller size (in hidden units)
help = 'The number of hidden units in the controller network'
parser.add_argument('--controller_size',
action='store',
dest='controller_size',
type=int,
help=help)
# The Learning Rate
help = 'The learning rate to use (in the form xe-y)'
parser.add_argument('--lr', action='store', dest='lr', type=float, help=help)
args = parser.parse_args()
# 1. setting up
opt = Options(**vars(args)) #unpack the arguments
np.random.seed(opt.seed)
# 2. env (prototype)
env_prototype = EnvDict[opt.env_type]
# 3. circuit (prototype)
circuit_prototype = CircuitDict[opt.circuit_type]
# 4. agent
agent = AgentDict[opt.agent_type](opt.agent_params,
env_prototype=env_prototype,
circuit_prototype=circuit_prototype)
if args.gpu == True:
circuit_prototype = torch.nn.DataParallel(circuit_prototype).cuda()
# 6. fit model
if opt.mode == 1: # train
import os
import time
import tensorflow as tf
from utils.options import Options
from data.dataset import Dataset
from models.cyclegan import CycleGANModel
tf.enable_eager_execution()
"""
Run this module for testing.
Required args: --data_dir, --save_dir, --results_dir
"""
if __name__ == "__main__":
opt = Options().parse(training=False)
dataset = Dataset(opt)
model = CycleGANModel(opt)
device = ("/gpu:" + str(opt.gpu_id)) if opt.gpu_id != -1 else "/cpu:0"
with tf.device(device):
start = time.time()
for image_index in range(opt.num_test):
model.set_input(dataset.data)
test_images = model.test()
dataset.save_images(test_images, image_index)
print(
"Generating {} test images for both datasets finished in {} sec\n".
format(opt.num_test,
time.time() - start))
import numpy as np
import os
import torch
import torch.nn as nn
import torch.multiprocessing as mp
from utils.options import Options
from utils.factory import GlobalLogsDict, ActorLogsDict, LearnerLogsDict, EvaluatorLogsDict
from utils.factory import LoggersDict, ActorsDict, LearnersDict, EvaluatorsDict, TestersDict
from utils.factory import EnvsDict, MemoriesDict, ModelsDict
if __name__ == '__main__':
mp.set_start_method("spawn", force=True)
opt = Options()
torch.manual_seed(opt.seed)
env_prototype = EnvsDict[opt.env_type]
memory_prototype = MemoriesDict[opt.memory_type]
model_prototype = ModelsDict[opt.model_type]
# dummy env to get state/action/reward/gamma/terminal_shape & action_space
dummy_env = env_prototype(opt.env_params, 0)
opt.norm_val = dummy_env.norm_val # use the max val of env states to normalize model inputs
opt.state_shape = dummy_env.state_shape
opt.action_shape = dummy_env.action_shape
opt.action_space = dummy_env.action_space
opt.reward_shape = opt.agent_params.num_tasks
opt.gamma_shape = opt.agent_params.num_tasks
opt.terminal_shape = opt.agent_params.num_tasks
del dummy_env
train_dl = data.create_data_loader(opt, "train")
test_dl = data.create_data_loader(opt, "test")
criterion = getattr(common, opt.criterion)(*opt.criterion_args)
temperature = TemperatureScheduler(*opt.temperature)
optimizer = getattr(torch.optim, opt.optimizer)(model.parameters(), *opt.optimizer_args)
scheduler = getattr(torch.optim.lr_scheduler, opt.scheduler)(optimizer, *opt.scheduler_args)
device = torch.device(opt.device)
model, epoch, optimizer, scheduler = load_checkpoint(opt.checkpoint_path, model, optimizer, scheduler, device)
if opt.is_classification:
test_metric = test_accuracy
metric_name = 'Accuracy'
else: # segmentation
test_metric = partial(test_segmentation, n_classes=opt.n_classes)
metric_name = 'mIoU'
print('Setting up complete, starting training')
for ep in range(epoch + 1, opt.max_epoch+1):
train_epoch(ep, model, criterion, temperature, optimizer, train_dl, device, writer, batch_average=opt.batch_average)
test_score = test_metric(model, temperature.get(ep), test_dl, device)
writer.add_scalar(f"{metric_name}/test", test_score, ep * len(test_dl.dataset))
print(f"Test {metric_name} after {ep} epochs = {test_score}")
scheduler.step()
if ep % opt.save_freq == 0:
save_checkpoint(model, optimizer, scheduler, ep, opt)
if __name__ == '__main__':
options = Options(config_file_arg="config_path")
main(options)
with torch.no_grad():
for i in range(h // split_size + 1):
for j in range(w // split_size + 1):
endi, endj = (i + 1) * split_size, (j + 1) * split_size
patch = image[:, :, i * split_size: endi, j * split_size: endj]
patch = patch.contiguous()
predicted = model.module.tta_eval(patch)
predicted = predicted * 255.0
mask = predicted.cpu().squeeze().detach().numpy()
mask = np.round(mask).astype(np.uint8)
result[i * split_size: endi, j * split_size: endj] = mask
return result
if __name__ == "__main__":
args = Options().parse()
if "simple" in str.lower(args.dataset):
dataset = "simple"
else:
assert "complex" in str.lower(args.dataset)
dataset = "complex"
transform = transforms.Compose([
transforms.ToTensor()
])
testset = URISC(path=args.dataset, mode="test", transform=transform)
testloader = DataLoader(testset, batch_size=1, shuffle=False)
if args.model == "CASENet":
model = DataParallel(CASENet(backbone=args.backbone)).cuda()
elif args.model == "ResNetUNet":
import tensorflow as tf import numpy as np from env.preon_env import Preon_env from utils.options import Options import tflearn import sys import os from networks import ActorNetwork, CriticNetwork from replay_buffer import ReplayBuffer # ========================== # Training Parameters # ========================== opt = Options() # Max training steps MAX_EPISODES = 50000 # Base learning rate for the Actor network ACTOR_LEARNING_RATE = opt.agent_params.actor_lr # Base learning rate for the Critic Network CRITIC_LEARNING_RATE = opt.agent_params.critic_lr # Discount factor GAMMA = opt.agent_params.gamma # Soft target update param TAU = opt.agent_params.tau # =========================== # Utility Parameters
#!/usr/bin/env python
# ! -*- coding: utf-8 -*-
"""
Main entry point.
Starts the device detection with notification service.
"""
from notification.telegram_service import TelegramService
from utils.options import Options
from device_detection.device_detector import DeviceDetector
if __name__ == "__main__":
options = Options.get_cli_options()
telegram_service = TelegramService()
detection = DeviceDetector(
device_address=options.device,
minimum_absence_for_notification=options.absence,
device_scan_interval=options.scan_interval,
notification_service=telegram_service)
detection.run()
