def test_allTest(self):
linux_Test = unittest.TestSuite()
linux_Test.addTests([
unittest.defaultTestLoader.loadTestsFromTestCase(Authentication),
unittest.defaultTestLoader.loadTestsFromTestCase(UserProfile),
unittest.defaultTestLoader.loadTestsFromTestCase(Projects),
unittest.defaultTestLoader.loadTestsFromTestCase(AccountSettings),
unittest.defaultTestLoader.loadTestsFromTestCase(OrderList),
unittest.defaultTestLoader.loadTestsFromTestCase(TryFree),
unittest.defaultTestLoader.loadTestsFromTestCase(JobPlacement),
unittest.defaultTestLoader.loadTestsFromTestCase(WorkExperience),
unittest.defaultTestLoader.loadTestsFromTestCase(Groupclass),
])
with open(direct + "/TestResult-Int.html", "w") as outfield:
runner1 = HTMLTestRunner.HTMLTestRunner(
stream=outfield,
title="LinuxJobber Test Report",
description="New Automated Test for the Main FUnctions of the Linuxjobber Site"
)
runner1.run(linux_Test)
# todo Add a mail notification when the script runs finish
mailer = get_mail_client()
parser = get_parser()
site = parser.get('site_to_test', 'url')
recipient_list = parser.get('mail', 'recipients')
from utils import send_email_ses
send_email_ses('Int', recipient_list)
def main():
# loads a lot of default parser values from the 'parser' file
parser = get_parser()
# get args from parser as an object
args = parser.parse_args()
args.device = 'cuda' if args.cuda else 'cpu'
# initialize seeds
utils.init_seed(args.seed)
# print('loader stuff', args)
loader = Loader.IncrementalLoader(args, seed=args.seed)
# print('loader stuff after after', args)
n_inputs, n_outputs, n_tasks = loader.get_dataset_info()
# setup logging
# logging is from 'misc_utils.py' from 'utils' folder
timestamp = utils.get_date_time() # this line is redundant bcz log_dir already takes care of it
args.log_dir, args.tf_dir = utils.log_dir(args, timestamp) # stores args into "training_parameters.json"
# create the model neural net
model = Model.Net(n_inputs, n_outputs, n_tasks, args, innerlr=args.opt_lr, outerlr=args.alpha_init)
# make model cuda-ized if possible
model.net.to(args.device)
# for all the CL baselines
result_val_t, result_val_a, result_test_t, result_test_a, spent_time = life_experience(model, loader, args)
# save results in files or print on terminal
save_results(args, result_val_t, result_val_a, result_test_t, result_test_a, model, spent_time)
def run_dataloader():
"""test dataloader"""
parser = get_parser()
# add model specific args
parser = BertLabeling.add_model_specific_args(parser)
# add all the available trainer options to argparse
# ie: now --gpus --num_nodes ... --fast_dev_run all work in the cli
parser = Trainer.add_argparse_args(parser)
args = parser.parse_args()
args.workers = 0
args.default_root_dir = "train_logs/debug"
model = BertLabeling(args)
from tokenizers import BertWordPieceTokenizer
tokenizer = BertWordPieceTokenizer(
os.path.join(args.bert_config_dir, "vocab.txt"))
loader = model.train_dataloader()
for d in loader:
# print(d)
input_ids = d[0][0].tolist()
match_labels = d[-1][0]
# start_positions, end_positions = torch.where(match_labels > 0)
# start_positions = start_positions.tolist()
# end_positions = end_positions.tolist()
# if not start_positions:
# continue
print("=" * 20)
# print(input_ids)
print(tokenizer.decode(input_ids, skip_special_tokens=False))
exit()
def main():
# run_dataloader()
"""main"""
# '''
parser = get_parser()
# add model specific args
parser = BertLabeling.add_model_specific_args(parser)
# add all the available trainer options to argparse
# ie: now --gpus --num_nodes ... --fast_dev_run all work in the cli
parser = Trainer.add_argparse_args(parser)
args = parser.parse_args()
model = BertLabeling(args)
if args.pretrained_checkpoint:
model.load_state_dict(
torch.load(args.pretrained_checkpoint,
map_location=torch.device('cpu'))["state_dict"])
checkpoint_callback = ModelCheckpoint(
filepath=args.default_root_dir,
save_top_k=2,
verbose=True,
monitor="coach_f1",
period=-1,
mode="max",
)
early_stop_callback = EarlyStopping(monitor="coach_f1",
patience=args.early_stop,
verbose=True,
mode="max",
min_delta=0.00)
trainer = Trainer.from_argparse_args(
args,
checkpoint_callback=checkpoint_callback,
callbacks=[early_stop_callback])
if not args.only_test:
trainer.fit(model)
print(checkpoint_callback.best_model_path)
# test
model = BertLabeling.load_from_checkpoint(
checkpoint_path=checkpoint_callback.best_model_path,
map_location=None,
batch_size=16,
max_length=128,
workers=0)
trainer.test(model=model)
# test on seen and unseen
print("**********testing on unseen data**********")
dataset_seen, dataset_unseen = get_dataloader_test(
model.args.tgt_domain, tokenizer=model.tokenizer)
model.dataset_test = dataset_unseen
trainer.test(model=model)
print("**********testing on unseen data**********")
model.dataset_test = dataset_seen
trainer.test(model=model)
def benchmark_parsing_methods():
"""Checks whether usage of lxml speed ups parsing"""
global dates
dates = dates[:]
html_parser = utils.get_parser("bgp")(parser="html.parser")
lxml_parser = utils.get_parser("bgp")(parser="lxml")
html_start = time.time()
c = [html_parser.get_currency(currency_name="USD", date=d) for d in dates]
html_finish = time.time()
lxml_start = time.time()
# TODO: network factor affects a lot, perhaps we should preliminary
# get all response texts and then test parsing with them
c = [lxml_parser.get_currency(currency_name="USD", date=d) for d in dates]
lxml_finish = time.time()
print("LXML avg. time: {}".format((lxml_finish - lxml_start) / len(dates)))
print("HTML avg. time: {}".format((html_finish - html_start) / len(dates)))
def main():
parser = get_parser("test")
args = parser.parse_args()
print(args)
cuda = torch.cuda.is_available()
node_dict, edge_dict, text_dict = load_dict(args)
test_data = load_data(args, node_dict, edge_dict, text_dict,
stage="test")
test_tgt_sizes = [test_data.item_size(i, -1) for i in range(len(test_data))]
print(" [test]: {} examples".format(len(test_data)))
test_iters = DataLoader(test_data,
batch_sampler=BatchSampler(torch.tensor(test_tgt_sizes), batch=args.batch_size),
collate_fn=test_data.collate_fn)
model = GraphTrans(args, node_dict, edge_dict, text_dict)
model.eval()
if cuda:
model.cuda()
saved = load_model(args, model, inference=True)
if not saved:
raise FileNotFoundError("Checkpoint does not exist")
edges_correct, edges_num, edges_pred = 0, 0, 0
nodes_correct, nodes_num, nodes_pred = 0, 0, 0
graphs, graph_corrects = 0, 0
for i, test_it in enumerate(test_iters):
if cuda:
samples = move_to_cuda(test_it)
else:
samples = test_it
batch_correct, batch_num, batch_pred, batch_graph_correct = greedy_search(model, samples["src_graph"], samples["src_text"], samples["tgt_graph"],
node_dict, edge_dict, args.max_nodes, cuda)
nodes_correct += batch_correct[0]
nodes_num += batch_num[0]
nodes_pred += batch_pred[0]
edges_correct += batch_correct[1]
edges_num += batch_num[1]
edges_pred += batch_pred[1]
graph_corrects += batch_graph_correct
graphs += 1
print("Node: Recall: {:.2f}({}/{}), Precision: {:.2f}({}/{}) ".format(nodes_correct/nodes_num * 100, nodes_correct, nodes_num, nodes_correct/nodes_pred * 100, nodes_correct, nodes_pred))
print("Edge: Recall: {:.2f}({}/{}), Precision: {:.2f}({}/{}) ".format(edges_correct/edges_num * 100, edges_correct, edges_num, edges_correct/edges_pred * 100, edges_correct, edges_pred))
print("Accuracy: {:.2f}({}/{})".format(graph_corrects/graphs * 100, graph_corrects, graphs))
def benchmark_parsing_methods():
"""Checks whether usage of lxml speed ups parsing"""
global dates
dates = dates[:]
html_parser = utils.get_parser("bgp")(parser="html.parser")
lxml_parser = utils.get_parser("bgp")(parser="lxml")
html_start = time.time()
c = [html_parser.get_currency(currency_name="USD",
date=d)
for d in dates]
html_finish = time.time()
lxml_start = time.time()
# TODO: network factor affects a lot, perhaps we should preliminary
# get all response texts and then test parsing with them
c = [lxml_parser.get_currency(currency_name="USD",
date=d)
for d in dates]
lxml_finish = time.time()
print("LXML avg. time: {}".format((lxml_finish - lxml_start) / len(dates)))
print("HTML avg. time: {}".format((html_finish - html_start) / len(dates)))
def main(config):
parser = get_parser()
argv = sys.argv[1:]
args, _ = parser.parse_known_args(argv)
init_logging_handler(config.log_dir)
logging.info(args)
config = update_cfg(config, args)
logging.info("Start initializing")
irl_model = RewardModule(config).to(device=device) # this is the reward model only, which will be fed to RewardEstimator.
reward_agent = RewardEstimator(config=config, irl_model=irl_model)
user_policy = ActorCriticDiscrete(config).to(device=device)
user_policy = init_net(user_policy)
user_ppo = PPO(config, user_policy)
system_policy = ActorCriticContinuous(config).to(device=device)
system_policy = init_net(system_policy)
init_system_policy = ActorCriticContinuous(config).to(device=device)
init_system_policy.load_state_dict(system_policy.state_dict())
system_ppo = PPO(config, system_policy, init_policy=init_system_policy)
# reward_true = RewardTruth(config).to(device=device) # this is the ground truth which will not be updated once randomly initialized.
reward_true = RewardTruthSampled(config).to(device)
reward_true = init_net(reward_true)
logging.info("Finish building module: reward agent, user ppo, system ppo")
main_agent = InteractAgent(config=config,
user_agent=user_ppo,
user_reward=reward_agent,
system_agent=system_ppo,
reward_groundtruth=reward_true
)
for e_id in range(config.master_epochs):
main_agent.master_train(e_id)
# for _ in range(3):
# main_agent.system_train()
# raise ValueError("stop here")
logging.info("@@@@@@@@@@ Finished @@@@@@@@@@@")
# import holoviews as hv
import matplotlib.pyplot as plt
from torchvision import datasets, transforms
import time
import os
import pickle
import shutil
import math
from training import mnist_training, mnist_testing
from mine_training import mi_aamine, mi_mine
from plots import plot_information_plane, plot_line
from utils import get_parser, Create_Logger
if __name__ == "__main__":
args = get_parser()
logger = Create_Logger(__name__)
# arguments passing
mnist_epochs = args.mnist_epoch
batch_size = args.batch_size
retrain = args.retrain
batch_group = args.batch_group
opt = args.mnist_opt
lr = args.mnist_lr
show = args.show
noise_var = args.noise_var
n_epoch = args.mine_epoch
aan_epoch = args.aamine_epoch
folder_name = args.folder_name
def get_parser(parser=None):
# default parameters for various projects
if parser is None:
parser = utils.get_parser()
parser.add_argument('--num_classes', default=None, type=int)
parser.add_argument('--input_size', default=None, type=int)
# custom parameters for quantization related projects
parser.add_argument('--base',
default=1,
type=int,
help='base used in GroupNet')
parser.add_argument('--width_alpha',
default=1.0,
type=float,
help='channel alpha')
parser.add_argument('--block_alpha', default=1.0, type=float)
parser.add_argument('--se_reduction',
default=16,
type=int,
help='ratio in Squeeze-Excition Module')
parser.add_argument('--stem_kernel', default=1, type=int)
parser.add_argument('--order', default='none', type=str)
parser.add_argument('--policy', default='none', type=str)
# config for activation quantization
parser.add_argument('--fm_bit', default=None, type=float)
parser.add_argument('--fm_level',
default=None,
type=int,
help="default of quantization level=2^bit - 1")
parser.add_argument('--fm_half_range',
action='store_false',
default=True,
help='real domain or non-positive range')
parser.add_argument('--fm_separator', default=0.38, type=float)
parser.add_argument('--fm_correlate', default=-1, type=float)
parser.add_argument('--fm_ratio', default=1, type=float)
parser.add_argument('--fm_scale', default=0.5, type=float)
parser.add_argument('--fm_enable',
action='store_true',
default=False,
help='enable quantization or not')
parser.add_argument('--fm_boundary', default=None, type=float)
parser.add_argument('--fm_quant_group', default=None, type=int)
# advanced options for gradient control / normalization / debug
parser.add_argument(
'--fm_adaptive',
default='none',
type=str,
choices=['none', 'var', 'mean', 'mean-var', 'var-mean'])
parser.add_argument('--fm_custom',
default='none',
type=str,
choices=['none', 'channel', 'resolution'])
parser.add_argument('--fm_grad_type',
default='none',
type=str,
choices=['none', 'STE', 'Triangle', 'STE-scale'])
parser.add_argument('--fm_grad_scale',
default='none',
type=str,
choices=[
'none', 'fan-scale', 'scale-fan', 'element-scale',
'scale-element'
])
# config for weight quantization
parser.add_argument('--wt_bit', default=None, type=float)
parser.add_argument('--wt_level', default=None, type=int)
parser.add_argument('--wt_half_range', action='store_true', default=False)
parser.add_argument('--wt_separator', default=0.38, type=float)
parser.add_argument('--wt_correlate', default=-1, type=float)
parser.add_argument('--wt_ratio', default=1, type=float)
parser.add_argument('--wt_scale', default=0.5, type=float)
parser.add_argument('--wt_enable', action='store_true', default=False)
parser.add_argument('--wt_boundary', default=None, type=float)
parser.add_argument('--wt_quant_group', default=None, type=int)
parser.add_argument(
'--wt_adaptive',
default='none',
type=str,
choices=['none', 'var', 'mean', 'mean-var', 'var-mean'])
parser.add_argument('--wt_grad_type',
default='none',
type=str,
choices=['none', 'STE', 'STE-scale'])
parser.add_argument('--wt_grad_scale',
default='none',
type=str,
choices=[
'none', 'fan-scale', 'scale-fan', 'element-scale',
'scale-element'
])
# config for output quantization
parser.add_argument('--ot_bit', default=None, type=float)
parser.add_argument('--ot_level', default=None, type=int)
parser.add_argument('--ot_half_range', action='store_true', default=False)
parser.add_argument('--ot_separator', default=0.38, type=float)
parser.add_argument('--ot_correlate', default=-1, type=float)
parser.add_argument('--ot_ratio', default=1, type=float)
parser.add_argument('--ot_scale', default=0.5, type=float)
parser.add_argument('--ot_enable', action='store_true', default=False)
parser.add_argument('--ot_boundary', default=None, type=float)
parser.add_argument('--ot_quant_group', default=None, type=int)
parser.add_argument(
'--ot_adaptive',
default='none',
type=str,
choices=['none', 'var', 'mean', 'mean-var', 'var-mean'])
parser.add_argument('--ot_grad_type',
default='none',
type=str,
choices=['none', 'STE', 'STE-scale'])
parser.add_argument('--ot_grad_scale',
default='none',
type=str,
choices=[
'none', 'fan-scale', 'scale-fan', 'element-scale',
'scale-element'
])
parser.add_argument('--ot_independent_parameter',
action='store_true',
default=False,
help="independent or shared parameters")
# re-init the model to pre-calculate some initial value
parser.add_argument('--re_init', action='store_true', default=False)
# proxquant
parser.add_argument('--proxquant_step', '--ps', default=5, type=int)
# mixup data augment
parser.add_argument('--mixup_alpha', default=0.7, type=float)
parser.add_argument('--mixup_enable', default=False, action='store_true')
parser.add_argument('--padding_after_quant',
action='store_true',
default=False)
# record / debug runtime information
parser.add_argument('--probe_iteration', default=1, type=int)
parser.add_argument('--probe_index', default=[], type=int, nargs='+')
parser.add_argument('--probe_list', default='', type=str)
# label-smooth
parser.add_argument('--label_smooth',
type=float,
default=0.1,
help='label smoothing')
# specific custom learning rate or weight decay for certain parameters
parser.add_argument('--custom_decay_list', default='', type=str)
parser.add_argument('--custom_decay', default=0.02, type=float)
parser.add_argument('--custom_lr_list', default='', type=str)
parser.add_argument('--custom_lr', default=1e-5, type=float)
# gloabl buffer
parser.add_argument('--global_buffer', default=dict(), type=dict)
return parser
import warnings
import torch
import datasets
import models
import training
import utils
from pathlib import Path
if __name__ == "__main__":
warnings.simplefilter(action="ignore", category=FutureWarning)
args = utils.get_parser().parse_args()
config = utils.load_config(args.config)
global_params = config["globals"]
output_dir = Path(global_params["output_dir"])
config_name = Path(args.config).name.replace(".yml", "")
output_dir = output_dir / config_name
if isinstance(global_params["seed"], list):
output_dir = output_dir / "multirun"
seeds = global_params["seed"]
multirun = True
else:
seeds = [global_params["seed"]]
multirun = False
output_dir.mkdir(parents=True, exist_ok=True)
# Add the created objects in the args namespace
args.env = env
args.policy_evaluation = policy_evaluation
args.policy_improvement = policy_improvement
args.experience_replay = experience_replay
args.tester = tester
args.log = log
if args.prioritized:
args.priority_update = priority_update_cb
# print the args
print_namespace(args)
# start the training
train(args)
if __name__ == "__main__":
main(
get_parser(
game="pong",
step_no=4_000_000,
update_freq=1,
learn_start=256,
mem_size=100_000,
epsilon_steps=30000,
lr=0.0001,
log_freq=5,
))
def run():
# Load commandline args and settings
parser = utils.get_parser()
args = parser.parse_args()
config = utils.get_config(args.settings)
# Canvas object and students classes
canvas = Canvas(config['url'], config['api'])
courses = config['classes']
days = args.days if args.weeks == 0 else args.weeks * 7
if args.reset:
print('Resetting output space')
config['longest_desc'] = 0
config['longest_clas'] = 0
for course_number in courses:
course = canvas.get_course(course_number)
# print(str(course)[12:])
'''This is working from here down'''
assignments = course.get_assignments()
for a in assignments:
try:
assignment_name = str(a).split('(')[0]
due_date, day_of_week = utils.get_date(a.due_at)
out = ""
if due_date > -2 and due_date < days:
prct = float(due_date) / float(days)
if prct < .25:
out += utils.URGENT
elif prct < .5:
out += utils.HIGH
elif prct < .75:
out += utils.MEDIUM
co = str(course)
co = co.split(' ')
if len(a.name) > config['longest_desc'] and len(
a.name) < 10:
config['longest_desc'] = len(a.name)
if len(co[1]) > config['longest_clas'] and len(co[1]) < 10:
config['longest_clas'] = len(co[1])
out += f"{co[2][0:config['longest_desc']]: <{config['longest_clas']}} | "
out += f"{str(a.name)[0:config['longest_desc']]: <{config['longest_desc']}} | "
out += f"{due_date + 1: <{2}} days left | "
out += f"{day_of_week: <9} | "
out += f"{a.html_url}" + utils.LOW
utils.todo_hw[due_date].append(out)
except Exception as e:
# print(e) # this will occur if the due date is None
pass
# Print the Homework assignments
i = 1
print(
f" {str('Course'): <{config['longest_clas']}} | {str('Homework'): <{config['longest_desc']}} | Days Left | Day | Link to Assignment"
)
utils.print_assignmentes()
if args.announcements != config[
"announcements"] and args.announcements is not None: # 5 is default
config["announcements"] = args.announcements
if args.reset:
print()
print('Re-run program for newly formatted output')
print(f'\n---Announcements for last {config["announcements"]} days---\n')
for course_number in courses:
course = canvas.get_course(course_number)
announcments = course.get_discussion_topics(only_announcements=True)
for i in announcments:
time_till_due, day_of_week = utils.get_date(i.created_at)
co = str(course)
co = co.split(' ')
out = ""
# print(f"config['announcements']*-1: {config['announcements']*-1}")
# print(f'time_till_due: {time_till_due}')
if time_till_due >= config['announcements'] * -1:
out += f'{str(co[2])[0:config["longest_clas"] + 4]: <{config["longest_clas"]}} | '
out += f'{(str(i)[0:config["longest_desc"] - 3] + "..."): <{config["longest_desc"]}} | '
out += f'{(time_till_due * -1)} days ago | {i.html_url}'
utils.todo_an[time_till_due].append(out)
utils.print_announcements()
c = 0
emails = canvas.get_conversations()
print(f'\n--- Emails ---\n')
for i in emails:
out = f'{str(c)+".":<3} '
if i.workflow_state == 'read':
out += f"{utils.MEDIUM}"
else:
out += f"{utils.URGENT}"
days, _ = utils.get_date(i.last_message_at)
out += f"{str(i.subject if len(i.subject) < 15 else str(i.subject)[0:12]+'...'):<15} | "
out += f"{str(i.participants[0]['name']if len(i.participants[0]['name']) < 15 else str(i.participants[0]['name'])[0:12]+'...'):<15} | "
out += f"{str((days*-1) - 2):<3} days ago | "
out += f"{str(i.context_name).split(' ')[2]:<13} | "
out += f"https://usu.instructure.com/conversations#filter=type=inbox"
out += utils.LOW
print(out)
c += 1
if c > 10:
break
utils.save_settings(config, args.settings)
def get_image_link(url):
info_soup = get_parser(url)
for image in info_soup.find_all('a', {'class': 'image'}):
image_link = 'https://en.wikipedia.org' + image.get('href')
print(image_link)
def main():
p = get_parser()
args = p.parse_args()
session = keystoneutils.get_session(args)
conf = ConfigHelper(path=os.path.join(os.environ['HOME'], '.cicd.conf'))
config = conf.get_conf()
ansible_path = config['ansible']['ansible_path']
ansible_extra_path = config['ansible']['ansible_extra']
if not os.path.isdir(os.path.join(ansible_path, 'cicd')):
os.mkdir(os.path.join(ansible_path, 'cicd'))
if not os.path.isdir(os.path.join(ansible_path, 'cicd/pre')):
os.mkdir(os.path.join(ansible_path, 'cicd/pre'))
if not os.path.isdir(os.path.join(ansible_path, 'cicd/pre/fstab')):
os.mkdir(os.path.join(ansible_path, 'cicd/pre/fstab'))
# MySQL Backup
client = paramiko.client.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.load_system_host_keys()
client.connect(hostname='10.41.3.201', username='******',
password='******')
client.exec_command(
'sudo su dba; /home/dba/admin/scripts/db_bkp_xtrabkp.sh -cq')
# Hostfile Diff
old_hostfile = read_hostfile(os.path.join(ansible_extra_path, 'hostfile'))
new_hostfile = read_hostfile(os.path.join(ansible_path, 'hostfile'))
unchanged_hosts, deleted_hosts, new_hosts = hostfile_diff(new_hostfile,
old_hostfile)
print_diff_hostfile(unchanged_hosts, deleted_hosts, new_hosts)
# package details
hostnames = get_representative(new_hostfile)
hostnames = list(set(hostnames) - set(new_hostfile['mysql']) - set(new_hostfile['mons']))
username = '******'
parallel_get_apt_list(hostnames, username, ansible_path)
# fstab
parallel_sftp(get_all(new_hostfile['connet']),
os.path.join(ansible_path, 'cicd/pre/fstab'))
parallel_sftp(get_all(new_hostfile['compute']),
os.path.join(ansible_path, 'cicd/pre/fstab'))
# Collect detailed information about all the VMs in SDCloud
nc = novautils.get_client(session)
vms = get_vm_list(nc=nc, filename=os.path.join(ansible_path,
'cicd/pre/vm_info.json'))
print '%s%s%s\n' % (Color.BOLD, 'VM States', Color.NORMAL)
vm_state_list = get_vm_state_count(vms)
for state in vm_state_list:
print '%s%s: %s%s' % (vm_state_list[state]['color'], state,
vm_state_list[state]['count'], Color.NORMAL)
print 'Total VMs: %s' % len(vms)
# YAML Diff
print '\n'
# base_vars.yml
print '\n'
old_basevars = read_yaml(os.path.join(ansible_extra_path, 'base_vars.yml'))
new_basevars = read_yaml(os.path.join(ansible_path, 'base_vars.yml'))
print '%s%s%sbase_vars.yml%s' % (Color.WHITE, Color.ON_BLACK, Color.BOLD,
Color.NORMAL)
unchanged_vars, changed_vars, deleted_vars, new_vars = yaml_diff(
new_basevars, old_basevars)
print_diff_yaml(new_basevars, old_basevars, unchanged_vars, changed_vars,
deleted_vars, new_vars)
# my_vars.yml
print '\n'
old_myvars = read_yaml(os.path.join(ansible_extra_path, 'my_vars.yml'))
new_myvars = read_yaml(os.path.join(ansible_path, 'my_vars.yml'))
print '%s%s%smy_vars.yml%s' % (Color.WHITE, Color.ON_BLACK, Color.BOLD,
Color.NORMAL)
unchanged_vars, changed_vars, deleted_vars, new_vars = yaml_diff(
new_myvars, old_myvars)
print_diff_yaml(new_myvars, old_myvars, unchanged_vars, changed_vars,
deleted_vars, new_vars)
# cinder_vars.yml
print '\n'
old_myvars = read_yaml(os.path.join(ansible_extra_path, 'cinder_vars.yml'))
new_myvars = read_yaml(os.path.join(ansible_path, 'cinder_vars.yml'))
print '%s%s%scinder_vars.yml%s' % (Color.WHITE, Color.ON_BLACK, Color.BOLD,
Color.NORMAL)
unchanged_vars, changed_vars, deleted_vars, new_vars = yaml_diff(
new_myvars, old_myvars)
print_diff_yaml(new_myvars, old_myvars, unchanged_vars, changed_vars,
deleted_vars, new_vars)
from __future__ import print_function
import numpy as np
import theano
np.random.seed(1337)
from keras.optimizers import RMSprop
from keras.models import Graph
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution1D, MaxPooling1D
import sys,json
import cPickle as pickle
from utils import get_parser
from load_data import load_unlabeled
parser = get_parser()
p = parser.parse_args()
(X_train, Y_train, _) = load_unlabeled()
X_train = np.asarray(X_train, dtype = theano.config.floatX).reshape((X_train.shape[0],X_train.shape[1],1))
Y_train = np.asarray(Y_train, dtype = bool)
index = range(X_train.shape[0])
np.random.shuffle(index)
X_train = X_train[index]
Y_train = Y_train[index]
Ydim = Y_train.shape[1]
Xdim = X_train.shape[1]
print('Build model...')
model = Graph()
) if args.experiment != 'logreg' else nn.BCELoss()
timings = []
for epoch in range(1, args.epochs + 1):
start = time.perf_counter()
dataloader = data.dataloader(train_data, train_labels, args.batch_size)
for batch_idx, (x, y) in enumerate(dataloader):
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
model.zero_grad()
outputs = model(x)
loss = loss_function(outputs, y)
with backpack(BatchGrad(), BatchL2Grad()):
loss.backward()
optimizer.step()
torch.cuda.synchronize()
duration = time.perf_counter() - start
print("Time Taken for Epoch: ", duration)
timings.append(duration)
if not args.no_save:
utils.save_runtimes(__file__.split('.')[0], args, timings)
else:
print('Not saving!')
print('Done!')
if __name__ == '__main__':
parser = utils.get_parser(dpsgd_kwargs.keys())
args = parser.parse_args()
main(args)
# We could have user timeit but I'm afraid bank will ban us for that
import time
import random
from collections import deque
from concurrent.futures import ThreadPoolExecutor, as_completed
import utils
parser = utils.get_parser("bgp")()
NUMBER_OF_DATES = 20
date_diffs = list(range(10, NUMBER_OF_DATES + 1))
random.shuffle(date_diffs)
dates = [utils.get_date_from_date_diff(d) for d in date_diffs]
def result_date_saver(parser, currency, date):
return (date, parser.get_currency(currency, date))
def benchmark_multiple_downloads():
start = time.time()
c = [parser.get_currency(currency_name="USD", date=d) for d in dates]
finish = time.time()
linear_time = finish - start
q = deque()
start = time.time()
timings = []
for epoch in range(1, args.epochs + 1):
start = time.perf_counter()
model_obj.train(input_fn=train_input_fn, steps=steps_per_epoch)
duration = time.perf_counter() - start
print("Time Taken: ", duration)
timings.append(duration)
if args.dpsgd:
# eps = compute_epsilon(epoch, num_train_eg, args)
# print('For delta=1e-5, the current epsilon is: %.2f' % eps)
print('Trained with DPSGD optimizer')
else:
print('Trained with vanilla non-private SGD optimizer')
if not args.no_save:
utils.save_runtimes(__file__.split('.')[0], args, timings)
else:
print('Not saving!')
print('Done!')
if __name__ == '__main__':
parser = utils.get_parser(model_dict.keys())
parser.add_argument('--memory_limit', default=None, type=int)
parser.add_argument('--xla', dest='use_xla', action='store_true')
parser.add_argument('--no_xla', dest='use_xla', action='store_false')
parser.add_argument('--no_unroll', dest='no_unroll', action='store_true')
args = parser.parse_args()
main(args)
batch_loss_avg = 0
batch_idx = len(dataloader_tr)
logger.info("e%db%d Saving model...", epoch, batch_idx)
torch.save(
model.state_dict(), log_path + '/save/' + model_name + '_' +
str(epoch) + '_' + str(batch_idx) + '.pth')
eval_loss, class_iou, confusion_matrix = eval_set(dataloader_va)
eval_losses.append(eval_loss)
if lr_schedule_type == 'plateau':
scheduler.step(eval_loss)
logger.info("e%db%d Def learning rate: %s", epoch, batch_idx,
get_current_learning_rates()[0])
logger.info("e%db%d GNN learning rate: %s", epoch, batch_idx,
get_current_learning_rates()[1])
logger.info("e%db%d Eval loss: %s", epoch, batch_idx, eval_loss)
logger.info("e%db%d Class IoU:", epoch, batch_idx)
for cl in range(14):
logger.info("%+10s: %-10s" % (idx_to_label[cl], class_iou[cl]))
logger.info("Mean IoU: %s", np.mean(class_iou[1:]))
logger.info("e%db%d Confusion matrix:", epoch, batch_idx)
logger.info(confusion_matrix)
if __name__ == '__main__':
args = u.get_parser().parse_args()
main(args.filename)
"""
Returns a dataframe generated from dictionary or list of dictionaries.
"""
if isinstance(data, list):
return pd.DataFrame(data)
elif isinstance(data, dict):
return pd.DataFrame([data])
else:
return None
# end create_dataframe
###########################################################################
# M A I N S
###########################################################################
if __name__ == '__main__':
parser = get_parser()
parser.add_argument('--date', default='20191023', help='YYYYMMDD to get matchups')
parser.add_argument('--start', default='20191023', help='YYYYMMDD to get matchups start date')
parser.add_argument('--end', default='20191023', help='YYYYMMDD to get matchups end date')
args=parser.parse_args()
#matchups = get_matchups(datestr=args.date)
#print("Matchups for %s: \n%s" % (args.date, '\n'.join(matchups)))
all_matchups = get_matchups_range(start=args.start, end=args.end)
debug("\n%d Matchups for %s-%s: \n\n%s\n\n" % (len(all_matchups), args.start, args.end, str(','.join(all_matchups))))
from scores import save_game_csv
import multiprocessing as mp
MAX_PROCS = mp.cpu_count() - 1
pool = mp.Pool(MAX_PROCS)
def main():
parser = get_parser()
args = parser.parse_args()
print(args)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cuda = torch.cuda.is_available()
node_dict, edge_dict, text_dict = load_dict(args)
train_data = load_data(args, node_dict, edge_dict, text_dict)
dev_data = load_data(args, node_dict, edge_dict, text_dict, stage="dev")
train_tgt_sizes = [
train_data.item_size(i, -1) for i in range(len(train_data))
]
dev_tgt_sizes = [dev_data.item_size(i, -1) for i in range(len(dev_data))]
print(" [training]: {} examples".format(len(train_data)))
print(" [dev ]: {} examples".format(len(dev_data)))
train_iters = DataLoader(train_data,
batch_sampler=BatchSampler(
torch.tensor(train_tgt_sizes),
batch=args.batch_size),
collate_fn=train_data.collate_fn)
dev_iters = DataLoader(dev_data,
batch_sampler=BatchSampler(
torch.tensor(dev_tgt_sizes), batch=1),
collate_fn=dev_data.collate_fn)
model = GraphTrans(args, node_dict, edge_dict, text_dict)
print('| num. model params: {} (num. trained: {})'.format(
sum(p.numel() for p in model.parameters()),
sum(p.numel() for p in model.parameters() if p.requires_grad)))
if cuda:
model.cuda()
# opt = get_std_opt(args, model)
opt = torch.optim.Adam(model.parameters(), lr=1e-5)
print(model)
# best_val = 9e+99
best_val = 0.
start_epoch = 0
batch_step = 0
opt.zero_grad()
saved = load_model(args, model, optimizer=opt)
# load save model and optimizer from disk
if saved:
best_val = saved["best_val"]
# best_val = 0.
start_epoch = saved["epoch"]
total_steps = 0
for epoch in range(start_epoch, args.epochs):
model.train()
epoch_loss, epoch_steps = 0., 0
epoch_st = time.time()
for train_it in tqdm(list(train_iters)):
if cuda:
samples = move_to_cuda(train_it)
else:
samples = train_it
loss = model(samples["src_graph"], samples["src_text"],
samples["tgt_graph"])
loss = loss / args.accumulation_steps # Normalize our loss (if averaged)
epoch_loss += loss.item()
loss.backward()
if (
batch_step + 1
) % args.accumulation_steps == 0: # Wait for several backward steps
# opt.clip_grad_norm(args.clip_norm)
opt.step()
opt.zero_grad()
total_steps += 1
# total_steps = opt.get_step()
# evaluate the model on dev set
if total_steps % args.eval_step == 0:
val_acc = validation_acc(model, dev_iters, epoch,
args.epochs, node_dict, edge_dict,
10, cuda)
if val_acc > best_val:
save_model(args, model, opt, epoch, best_val, "best")
best_val = val_acc
epoch_steps += 1
batch_step += 1
val_acc = validation_acc(model, dev_iters, epoch, args.epochs,
node_dict, edge_dict, 10, cuda)
if val_acc > best_val:
save_model(args, model, opt, epoch, best_val, "best")
best_val = val_acc
save_model(args, model, opt, epoch + 1, best_val, "last")
epoch_time = (time.time() - epoch_st) / 60
train_info = "[Train {:02}/{:02}]: total_loss={:.4f} avg_loss={:.4f} total_steps={:05} elapse={:.4f} mins best_val={:.4f}"
print(
train_info.format(epoch + 1, args.epochs, epoch_loss,
epoch_loss / epoch_steps, total_steps,
epoch_time, best_val))
from cassandra_client import CassandraClient
from utils import create_song_by_name, get_song_by_name, get_parser, create_songs_played_by_user, \
get_songs_played_by_user, create_search_playlist_by_name, get_search_playlist_by_name, create_playlist_followers, \
get_playlist_followers, create_user_followers, get_user_followers, create_song_in_playlist, get_song_in_playlist, \
create_user_decreasing_popularity_prefix, get_user_in_decreasing_popularity, create_playlist_decreasing_popularity, \
get_playlist_decreasing_popularity
client = CassandraClient()
client.sync_models()
args = get_parser().parse_args()
ask = args.ask
if ask == 'song_by_name':
# Retrieve song by name
create_song_by_name(song_name='one1 song',
artist='one1 artist',
album='one1 album',
genre='one1 genre',
year=2100)
create_song_by_name(song_name='one2 song',
artist='one2 artist',
album='one2 album',
genre='one2 genre',
year=2200)
create_song_by_name(song_name='song3',
artist='one3 artist',
album='one3 album',
genre='one3 genre',
year=2009)
create_song_by_name(song_name='rock song',
# We could have user timeit but I'm afraid bank will ban us for that
import time
import random
from collections import deque
from concurrent.futures import ThreadPoolExecutor, as_completed
import utils
parser = utils.get_parser("bgp")()
NUMBER_OF_DATES = 20
date_diffs = list(range(10, NUMBER_OF_DATES + 1))
random.shuffle(date_diffs)
dates = [utils.get_date_from_date_diff(d) for d in date_diffs]
def result_date_saver(parser, currency, date):
return (date, parser.get_currency(currency, date))
def benchmark_multiple_downloads():
start = time.time()
c = [parser.get_currency(currency_name="USD",
date=d)
for d in dates]
finish = time.time()
linear_time = finish - start
q = deque()
# https://github.com/piskvorky/gensim/pull/231#issuecomment-59741971
import gensim
import numpy as np
import itertools
import nltk
import random
import utils
from bs4 import BeautifulSoup
from BioScope_word2vec import get_tokenized_sentences_from_Bioscope
if __name__ == "__main__":
######################## Parse command-line arguments #####################
parser = utils.get_parser()
# Add required argument of training data
parser.add_argument('filenames', metavar='filepath', type=str, nargs='+',
help='one or more XML files used as training data')
# Grab arguments from stdin
args = parser.parse_args()
# Convert parsed inputs into local variables
# filenames, min_count, verbose, debug, n_epochs
locals().update(args.__dict__)
min_word_count = min_count
DEBUG = debug
log.AvgMetric("rw_per_ep", emph=True),
log.AvgMetric("rw_per_step"),
log.MaxMetric("max_q"),
log.FPSMetric("training_fps"),
log.FPSMetric("sampling_fps"),
),
console_options=("white", "on_blue", ["bold"]),
)
log.log_info(train_log, "date: %s." % time.strftime("%d/%m/%Y | %H:%M:%S"))
log.log_info(train_log, "pytorch v%s." % torch.__version__)
# Add the created objects in the args namespace
args.env = env
args.policy_evaluation = policy_evaluation
args.policy_improvement = policy_improvement
args.experience_replay = experience_replay
args.tester = tester
args.log = log
if args.prioritized:
args.priority_update = priority_update_cb
# print the args
print_namespace(args)
# start the training
train(args)
if __name__ == "__main__":
main(get_parser())
def get_parameter():
parser = utils.get_parser()
# custom config for quantization
parser.add_argument('--keyword',
default='vgg16',
type=str,
help='key features')
parser.add_argument('--n_classes', default=21, type=int)
parser.add_argument(
'--aug',
default=None,
type=str,
)
parser.add_argument('--eval_flip', action='store_true', default=False)
parser.add_argument(
'--sbd',
default='benchmark_RELEASE',
type=str,
)
parser.add_argument(
'--val_split',
default='val',
type=str,
)
parser.add_argument(
'--train_split',
default='train_aug',
type=str,
)
parser.add_argument(
'--row',
default='same',
type=str,
)
parser.add_argument(
'--col',
default='same',
type=str,
)
parser.add_argument(
'--loss',
default='cross_entropy',
type=str,
)
parser.add_argument('--size_average', action='store_true', default=False)
parser.add_argument('--learned_billinear',
action='store_true',
default=False)
parser.set_defaults(batch_size=1)
parser.set_defaults(val_batch_size=1)
parser.set_defaults(model='fcn32s')
parser.set_defaults(dataset='pascal')
parser.set_defaults(root='/data/pascal')
parser.set_defaults(lr=1e-4) #fcn32s: 1e-10 /fcn16s: 1e-12/ fcn8s:1e-14/
parser.set_defaults(weight_decay=5e-4)
parser.set_defaults(momentum=0.9)
parser.set_defaults(lr_policy='fix')
parser.set_defaults(epochs=50)
args = parser.parse_args()
return args
def main():
parser = get_parser()
args = parser.parse_args()
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train and not args.overwrite_output_dir:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
args.output_dir))
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
args.device = device
if args.tpu:
if args.tpu_ip_address:
os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address
if args.tpu_name:
os.environ["TPU_NAME"] = args.tpu_name
if args.xrt_tpu_config:
os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config
assert "TPU_IP_ADDRESS" in os.environ
assert "TPU_NAME" in os.environ
assert "XRT_TPU_CONFIG" in os.environ
import torch_xla
import torch_xla.core.xla_model as xm
args.device = xm.xla_device()
args.xla_model = xm
# Setup logging
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
# Set seed
set_seed(args)
# preparing tokenizer
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case)
# Prepare data
args.task_name = args.task_name.lower()
args.output_mode = 'classification'
train_dataset, num_labels = load_build_dataset(args, tokenizer, evaluate=False)
eval_dataset = None
if args.do_eval:
eval_dataset, _ = load_build_dataset(args, tokenizer, evaluate=True)
# Load pretrained model
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
args.model_type = args.model_type.lower()
config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
num_labels=num_labels, finetuning_task=args.task_name)
model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path),
config=config)
if args.local_rank == 0:
torch.distributed.barrier()
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
# Create output directory if needed
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = model.module if hasattr(model,
'module') else model # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
# Load a trained model and vocabulary that you have fine-tuned
model = model_class.from_pretrained(args.output_dir)
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
model.to(args.device)
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else ""
model = model_class.from_pretrained(checkpoint)
model.to(args.device)
result = evaluate(args, model, eval_dataset, prefix=prefix)
result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())
results.update(result)
return results
