def __init__(self):
self.args = argparser()
propfiles = ['base.props']
if self.args.props_file:
propfiles.append(self.args.props_file)
self.props = properties(*propfiles, cmd_args=self.args.extra_props)
cluster.make_cluster_info(self.props)
self.bubble_thread = None
if self.args.log_path or self.args.output:
self.log_path = self.args.log_path or '.'
if self.log_path[-1] != '/':
self.log_path += '/'
self.log_filename = self.args.output or \
'epidemia_{}'.format(datetime.now().strftime('%Y%m%dT%H%M%S_%f')[:-3])
self.log_file = open(f'{self.log_path}{self.log_filename}.log',
'w')
else:
self.log_path = None
self.log_filename = None
self.log_file = None
# 'epoch' : t,
# 'lr_scheduler_dict': lr_scheduler.state_dict()
# }, model_name)
torch.save(model.state_dict(), model_name)
# save the best model after we reached the schedule
if t >= len(eps_schedule):
if err <= best_err:
best_err = err
recorded_clean_err = clean_err
logger.log('Saving best model {} with error {}'.format(
best_model_name, best_err))
torch.save(
{
'state_dict': model.state_dict(),
'opt_state_dict': opt.state_dict(),
'robust_err': err,
'clean_err': clean_err,
'epoch': t,
'lr_scheduler_dict': lr_scheduler.state_dict()
}, best_model_name)
logger.log('Total Time: {:.4f}'.format(timer))
logger.log('Model {} best err {}, clean err {}'.format(
model_id, best_err, recorded_clean_err))
if __name__ == "__main__":
args = argparser()
main(args)
def main(args=None):
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
Function: - Attempts to exit the program cleanly when called
'''
def Terminate(exitcode):
sys.exit(exitcode)
'''
This is the mainline section of the program and makes calls to the
various other sections of the code
'''
if __name__ == '__main__':
ret = 0
AP = argparser()
ret = AP.Parse(sys.argv)
if ret == -1:
Usage()
Terminate(ret)
LG = logreader()
LG.ConfRead(AP.debug)
AP.logdir = LG.logdir
AP.emailrecp = LG.emailrecp
AP.emailsend = LG.emailsend
if (len(LG.logdir) < 4):
print 'The log directory has not been configured. Please edit the pyrecon.conf file before continuing.'
print ''
def main(args=None):
try:
# if terminal
if config.IS_IN_TERMINAL:
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# if input stream
else:
COMMAND, ARGS = modifyArgsByStdinStream()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
def main(args=None):
os.environ['TMP'] = '/tmp'
os.environ['TEMP'] = '/tmp'
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
if isinstance(ARGS, dict) and ARGS.get('content') == '-':
# content from stdin!
content = sys.stdin.read()
ARGS['content'] = content
# error or help
if COMMAND is None or ARGS is False:
return tools.exitErr()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
if COMMAND == 'dedup':
Notes().dedup(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
def main(args=None):
try:
# if terminal
if config.IS_IN_TERMINAL:
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# if input stream
else:
COMMAND, ARGS = modifyArgsByStdinStream()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
from argparser import argparser
from arguments import Arguments
from fcn import FCN
from multi_class_hinge_loss import multiClassHingeLoss
import resnet
from svm import SVM
from train import test
from utils import accumulate_grads_over_epochs, \
get_testloader, get_trainloader, vec_unit_dot
import functools
print = functools.partial(print, flush=True)
# Setups
args = Arguments(argparser())
hook = sy.TorchHook(torch)
USE_CUDA = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device(
"cuda:{}".format(args.device_id[0]) if USE_CUDA else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if USE_CUDA else {}
kwargs = {}
ckpt_path = '../ckpts'
folder = '{}_{}'.format(args.dataset, 'centralized')
model_name = 'clf_{}_noise_{}_paradigm_{}' \
'_lr_{}_decay_{}_batch_{}_distributed_sim'.format(
args.clf, args.noise, '_'.join(args.paradigm),
def main(args=None):
os.environ['TMP'] = '/tmp'
os.environ['TEMP'] = '/tmp'
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
if isinstance(ARGS, dict) and ARGS.get('content') == '-':
# content from stdin!
content = sys.stdin.read()
ARGS['content'] = content
# error or help
if COMMAND is None or ARGS is False:
return tools.exitErr()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
if COMMAND == 'dedup':
Notes().dedup(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
from argparser import argparser_generic as argparser
from simulate_data import DataAD
from projgamma import GammaPrior
from pandas import read_csv
import models
import os
if __name__ == '__main__':
p = argparser()
Chain = models.Chains[p.model]
Result = models.Results[p.model]
data = DataAD(p.in_path)
if p.model.startswith('dp'):
emp_path = os.path.join(
p.out_folder,
p.model,
'empirical.csv',
)
out_path = os.path.join(
p.out_folder,
p.model,
'results_{}_{}.db'.format(p.eta_shape, p.eta_rate),
)
pp_path = os.path.join(
p.out_folder,
p.model,
'postpred_{}_{}.csv'.format(p.eta_shape, p.eta_rate),
)
_money = float(args.money)
days = int(args.days)
# calculate the given profit with the money.money you earned
money.money(args).profit(args.profit, _money, days)
# count command line parameter
countmoney = money.counter(args.count).calculate()
if args.count:
money.money(args).profit(args.countprofit, countmoney, len(args.count))
# predict adpack amount in x days
packs = adpack.adpacks(args.adpacks, args.balance, 0.0, 60.0,
float(args.course), args, date)
packs.predictAdpacks()
packs.predictTime()
if args.adpackprofit:
packs.AdpackProfit()
packs.targetProfit()
file.file(args).TotalEarnings()
if __name__ == '__main__':
g_date = date.date()
argument = argparser.argparser(g_date.date)
init(argument, g_date)
'''
Terminate()
Function: - Attempts to exit the program cleanly when called
'''
def Terminate(exitcode):
sys.exit(exitcode)
'''
This is the mainline section of the program and makes calls to the
various other sections of the code
'''
if __name__ == '__main__':
ret = 0
AP = argparser()
ret = AP.Parse(sys.argv)
if (ret == -1):
Usage()
Terminate(ret)
#Mix and match external classes
#Some duplication could be reduced in a later version
LOG = logger()
ret = LOG.ConfRead(AP.debug)
if (ret == -1):
print 'Logger terminated reading the configuration file...'
Terminate(ret)
def main(args=None):
try:
# if in terminal or if there are command line parameters
if len(sys.argv) > 1 or config.IS_IN_TERMINAL:
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
if isinstance(ARGS, dict) and ARGS["content"] == "-":
## content comes from stdin!
content = sys.stdin.read()
ARGS["content"] = content
# if input stream
else:
COMMAND, ARGS = modifyArgsByStdinStream()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == "user":
User().user(**ARGS)
if COMMAND == "login":
User().login(**ARGS)
if COMMAND == "logout":
User().logout(**ARGS)
if COMMAND == "settings":
User().settings(**ARGS)
# Notes
if COMMAND == "create":
Notes().create(**ARGS)
if COMMAND == "edit":
Notes().edit(**ARGS)
if COMMAND == "remove":
Notes().remove(**ARGS)
if COMMAND == "show":
Notes().show(**ARGS)
if COMMAND == "find":
Notes().find(**ARGS)
# Notebooks
if COMMAND == "notebook-list":
Notebooks().list(**ARGS)
if COMMAND == "notebook-create":
Notebooks().create(**ARGS)
if COMMAND == "notebook-edit":
Notebooks().edit(**ARGS)
if COMMAND == "notebook-remove":
Notebooks().remove(**ARGS)
# Tags
if COMMAND == "tag-list":
Tags().list(**ARGS)
if COMMAND == "tag-create":
Tags().create(**ARGS)
if COMMAND == "tag-edit":
Tags().edit(**ARGS)
if COMMAND == "tag-remove":
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
def main(args=None):
os.environ["TMP"] = "/tmp"
os.environ["TEMP"] = "/tmp"
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
if isinstance(ARGS, dict) and ARGS.get("content") == "-":
# content from stdin!
content = sys.stdin.read()
ARGS["content"] = content
# error or help
if COMMAND is None or ARGS is False:
return tools.exitErr()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == "user":
User().user(**ARGS)
if COMMAND == "login":
User().login(**ARGS)
if COMMAND == "logout":
User().logout(**ARGS)
if COMMAND == "settings":
User().settings(**ARGS)
# Notes
if COMMAND == "create":
Notes().create(**ARGS)
if COMMAND == "edit":
Notes().edit(**ARGS)
if COMMAND == "remove":
Notes().remove(**ARGS)
if COMMAND == "show":
Notes().show(**ARGS)
if COMMAND == "find":
Notes().find(**ARGS)
if COMMAND == "dedup":
Notes().dedup(**ARGS)
# Notebooks
if COMMAND == "notebook-list":
Notebooks().list(**ARGS)
if COMMAND == "notebook-create":
Notebooks().create(**ARGS)
if COMMAND == "notebook-edit":
Notebooks().edit(**ARGS)
if COMMAND == "notebook-remove":
Notebooks().remove(**ARGS)
# Tags
if COMMAND == "tag-list":
Tags().list(**ARGS)
if COMMAND == "tag-create":
Tags().create(**ARGS)
if COMMAND == "tag-edit":
Tags().edit(**ARGS)
if COMMAND == "tag-remove":
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
def main(args=None):
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# Didn't read any command line arguments, and not attached
# to an interactive tty?
# If so, look for input on piped stream
if COMMAND is None and not config.IS_IN_TERMINAL:
COMMAND, ARGS = modifyArgsByStdinStream()
if COMMAND is None:
out.printAbout()
# error or help
if COMMAND is None or ARGS is False:
return tools.exitErr()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
