def main():
argparser = ArgParser(description="Load TUPA model and visualize, saving to .png file.")
argparser.add_argument("models", nargs="+", help="model file basename(s) to load")
args = argparser.parse_args()
for filename in args.models:
model = load_model(filename)
visualize(model, filename)
def main():
parser = ArgParser(
default_config_files=['/etc/factoriomcd.ini', '~/.factoriomcd.ini'])
parser.add('-d', '--debug', action='store_true')
parser.add('-v', '--verbose', action='store_true')
parser.add('--log-file', default="/opt/factorio/server.out")
parser.add('--server-id', default="1")
parser.add('--rcon-host', default="localhost")
parser.add('--rcon-password', default="asdasd")
parser.add('--rcon-port', default=31337)
parser.add('--ws-url',
default="ws://127.0.0.1:8000/ws_v1/server_callback/1/")
parser.add('--ws-password', default="asdasd")
options = parser.parse_args()
if options.verbose:
coloredlogs.install(level='DEBUG')
logger.debug("FactorioMCd initializing...")
else:
coloredlogs.install(level='INFO')
FactorioMCd(options).run()
def create_args():
parser = ArgParser()
parser.add('--db_section')
parser.add('--reqnums')
parser.add('--csv')
args = parser.parse_args()
return args
def main():
argparser = ArgParser(
description=
"Load TUPA model and save the features enumeration as a text JSON file."
)
argparser.add_argument("models",
nargs="+",
help="model file basename(s) to load")
argparser.add_argument("-s",
"--suffix",
default=".enum.json",
help="filename suffix to append")
argparser.add_argument("-l",
"--lang",
help="use spaCy model to decode numeric IDs")
args = argparser.parse_args()
for filename in args.models:
model = load_model(filename)
params = model.feature_extractor.params
if args.lang:
vocab = get_vocab(lang=args.lang)
for param in params.values():
if param.data:
param.data = [
decode(vocab, v)
for v in sorted(param.data, key=param.data.get)
]
save_json(model.filename + args.suffix, params)
def __init__(self,chosen_dir=None):
#CMD arguments and configfile
if chosen_dir == None or not os.path.isdir(chosen_dir):
current_data_dir = os.getcwd()
parser = ArgParser(default_config_files=['par2deep.ini', '~/.par2deep'])
else:
current_data_dir = os.path.abspath(chosen_dir)
parser = ArgParser(default_config_files=[os.path.join(current_data_dir,'par2deep.ini'), '~/.par2deep'])
parser.add_argument("-q", "--quiet", action='store_true', help="Don't asks questions, go with all defaults, including repairing and deleting files (default off).")
parser.add_argument("-over", "--overwrite", action='store_true', help="Overwrite existing par2 files (default off).")
parser.add_argument("-novfy", "--noverify", action='store_true', help="Do not verify existing files (default off).")
parser.add_argument("-keepor", "--keep_orphan", action='store_true', help="Keep orphaned par2 files.")
#parser.add_argument("-seppardir", "--separate_parity_directory", action='store_true', help="Store parity data in a subdirectory.")
parser.add_argument("-clean", "--clean_backup", action='store_true', help="Remove backups created by par2 (.1,.2 and so on) from your file tree.")
parser.add_argument("-ex", "--excludes", action="append", type=str, default=[], help="Optionally excludes directories ('root' is files in the root of -dir).")
parser.add_argument("-exex", "--extexcludes", action="append", type=str, default=[], help="Optionally excludes file extensions.")
parser.add_argument("-dir", "--directory", type=str, default=current_data_dir, help="Path to protect (default is current directory).")
#parser.add_argument("-pardir", "--parity_directory", type=str, default=os.getcwd(), help="Path to parity data store (default is current directory).")
parser.add_argument("-pc", "--percentage", type=int, default=5, help="Set the parity percentage (default 5%%).")
parser.add_argument("-pcmd", "--par_cmd", type=str, default="", help="Set path to alternative par2 executable (default \"par2\").")
#lets get a nice dict of all o' that.
#FIXME: catch unrecognized arguments
args = {k:v for k,v in vars(parser.parse_args()).items() if v is not None}
args["nr_parfiles"] = str(1) #number of parity files
#set that shit
self.args = args
return
def main(cls, args: Optional[Dict[str, object]] = None):
if args is None:
arg_parser = ArgParser()
cls.add_arguments(arg_parser)
args = arg_parser.parse_args()
args = vars(args).copy()
if args.get("debug", False):
logging_level = logging.DEBUG
elif args.get("logging_level") is not None:
logging_level = getattr(logging, args["logging_level"].upper())
else:
logging_level = logging.INFO
logging.basicConfig(
format=
"%(asctime)s:%(module)s:%(lineno)s:%(name)s:%(levelname)s: %(message)s",
level=logging_level,
)
pipeline_kwds = args.copy()
for key in ("force", "force_extract", "logging_level"):
try:
pipeline_kwds.pop(key)
except KeyError:
pass
pipeline = cls(**pipeline_kwds)
force = bool(args.get("force", False))
force_extract = force or bool(args.get("force_extract", False))
pipeline.extract_transform_load(force_extract=force_extract)
def parse_args():
parser = ArgParser(default_config_files=[".env"])
parser.add("--start_page", required=True, type=int)
parser.add("--end_page", type=int, default=701)
parser.add("--file", required=True, help="books description file")
args = parser.parse_args()
return args
def parse_arguments() -> Arguments:
parser = ArgParser(default_config_files=['settings.conf'])
parser.add_argument('--parties-table', env_var='PARTIES_TABLE')
parser.add_argument('--client-secret-file',
default='secrets/client_secret.json',
env_var='CLIENT_SECRET_FILE')
parser.add_argument('--token-storage-file',
default='secrets/tokens.json',
env_var='TOKEN_STORAGE_FILE')
parser.add_argument('--html-template',
default='resources/email_template.html',
env_var='TEMPLATE',
type=_template)
parser.add_argument('--gimp-path',
default='/Applications/GIMP.app/Contents/MacOS/GIMP',
env_var='GIMP_PATH')
parser.add_argument('--gimp-template', env_var='GIMP_TEMPLATE')
parser.add_argument(
'--invitation-url',
env_var='RSVP_URL',
help='URL template for party RSVP. ' +
'Must include "{partyId}" and "{guestId}" placeholders, e.g. ' +
'"https://www.flyingjs4.life/invitation?party={partyId}&guest={guestId}"'
)
parser.add_argument('--sender',
env_var='SENDER',
help='Email address to use to send invitations.',
type=parse_email_address)
parser.add_argument('--envelopes-dir',
env_var='ENVELOPES_DIR',
help='Directory where envelope images will be saved.')
parser.add_argument(
'--resource-bucket',
env_var='RESOURCE_BUCKET',
help='Name of S3 bucket where static resources are stored.')
parser.add_argument(
'--envelope-prefix',
env_var='ENVELOPE_PREFIX',
help='Prefix of all envelope objects in resource bucket.')
parser.add_argument('--envelope-url-template',
env_var='ENVELOPE_URL_TEMPLATE',
help='URL template for envelope images. ' +
'Must include "{partyId}" placeholder, e.g. ' +
'"https://www.flyingjs4.life/envelopes/{partyId}.png"')
parser.add_argument(
'--send',
action='store_true',
help='Send emails. USE ONLY WHEN ABSOLUTELY READY TO INVITE EVERYONE!')
parser.add_argument('--skip-envelopes',
action='store_true',
help='Do not render envelopes')
parser.add_argument('--skip-email',
action='store_true',
help='Do not create or send email invitations')
parser.add_argument('--only',
action='append',
help='Only operate on the parties with these IDs',
default=[])
return Arguments(parser.parse_args())
def _initialize_arguments(p: configargparse.ArgParser):
p.add('--model_storage_directory', help='The directory caching all model runs')
p.add('--bert_model_path', help='Model path to BERT')
p.add('--labels', help='Numbers of labels to predict over', type=str)
p.add('--architecture', help='Training architecture', type=str)
p.add('--freeze_bert', help='Whether to freeze bert', type=bool)
p.add('--batch_size', help='Batch size for training multi-label document classifier', type=int)
p.add('--bert_batch_size', help='Batch size for feeding 510 token subsets of documents through BERT', type=int)
p.add('--epochs', help='Epochs to train', type=int)
#Optimizer arguments
p.add('--learning_rate', help='Optimizer step size', type=float)
p.add('--weight_decay', help='Adam regularization', type=float)
p.add('--evaluation_interval', help='Evaluate model on test set every evaluation_interval epochs', type=int)
p.add('--checkpoint_interval', help='Save a model checkpoint to disk every checkpoint_interval epochs', type=int)
#Non-config arguments
p.add('--cuda', action='store_true', help='Utilize GPU for training or prediction')
p.add('--device')
p.add('--timestamp', help='Run specific signature')
p.add('--model_directory', help='The directory storing this model run, a sub-directory of model_storage_directory')
p.add('--use_tensorboard', help='Use tensorboard logging', type=bool)
args = p.parse_args()
args.labels = [x for x in args.labels.split(', ')]
#Set run specific envirorment configurations
args.timestamp = time.strftime("run_%Y_%m_%d_%H_%M_%S") + "_{machine}".format(machine=socket.gethostname())
args.model_directory = os.path.join(args.model_storage_directory, args.timestamp) #directory
os.makedirs(args.model_directory, exist_ok=True)
#Handle logging configurations
log.handlers.clear()
formatter = logging.Formatter('%(message)s')
fh = logging.FileHandler(os.path.join(args.model_directory, "log.txt"))
fh.setLevel(logging.INFO)
fh.setFormatter(formatter)
log.addHandler(fh)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(formatter)
log.setLevel(logging.INFO)
log.addHandler(ch)
log.info(p.format_values())
#Set global GPU state
#if torch.cuda.is_available() and args.cuda:
# if torch.cuda.device_count() > 1:
# log.info("Using %i CUDA devices" % torch.cuda.device_count() )
# else:
# log.info("Using CUDA device:{0}".format(torch.cuda.current_device()))
#args.device = args.device #'cuda:1'
#else:
# log.info("Not using CUDA :(")
# args.dev = 'cpu'
return args
def main():
argparser = ArgParser(description="Load TUPA model and export as .npz file.")
argparser.add_argument("models", nargs="+", help="model file basename(s) to load")
args = argparser.parse_args()
for filename in args.models:
model = load_model(filename)
save_model(model, filename)
model.config.save(filename)
def main():
argparser = ArgParser(description="Load TUPA model and save again to a different file.")
argparser.add_argument("models", nargs="+", help="model file basename(s) to load")
argparser.add_argument("-s", "--suffix", default=".1", help="filename suffix to append")
args = argparser.parse_args()
for filename in args.models:
model = load_model(filename)
model.filename += args.suffix
model.classifier.filename += args.suffix
model.save()
def get_config() -> Namespace:
"""
Parse config and return the parsed args
:return: ConfigArgParse Namespace object holding parsed args
"""
parser = ArgParser()
add_args(parser)
args = parser.parse_args()
return args
class Cli:
__COMMAND_CLASSES = {
"create-spreadsheets": CreateSpreadsheetsCommand,
}
def __init__(self):
self.__arg_parser = ArgParser()
self.__logger = logging.getLogger(self.__class__.__name__)
def __add_arguments(self):
arg_parsers = [self.__arg_parser]
subparsers = self.__arg_parser.add_subparsers(dest="command", required=True)
for command_name, command_class in self.__COMMAND_CLASSES.items():
command_arg_parser = subparsers.add_parser(command_name)
command_class.add_arguments(command_arg_parser)
arg_parsers.append(command_arg_parser)
for arg_parser in arg_parsers:
arg_parser.add_argument("-c", is_config_file=True, help="config file path")
arg_parser.add_argument(
"--debug", action="store_true", help="turn on debugging"
)
arg_parser.add_argument(
"--logging-level",
help="set logging-level level (see Python logging module)",
)
def __configure_logging(self, args):
if args.debug:
logging_level = logging.DEBUG
elif args.logging_level is not None:
logging_level = getattr(logging, args.logging_level.upper())
else:
logging_level = logging.INFO
logging.basicConfig(
format="%(asctime)s:%(processName)s:%(module)s:%(lineno)s:%(name)s:%(levelname)s: %(message)s",
level=logging_level,
)
def main(self):
self.__add_arguments()
args = self.__arg_parser.parse_args()
self.__configure_logging(args)
command_class = self.__COMMAND_CLASSES[args.command]
command_kwds = vars(args).copy()
command_kwds.pop("c")
command_kwds.pop("logging_level")
command = command_class(**command_kwds)
command()
def main():
argparser = ArgParser(description="Visualize scores of a model over the dev set, saving to .png file.")
argparser.add_argument("models", nargs="+", help="model file basename(s) to load")
args = argparser.parse_args()
for pattern in args.models:
for filename in sorted(glob(pattern)) or [pattern]:
basename, _ = os.path.splitext(filename)
for div in "dev", "test":
try:
scores = load_scores(basename, div=div)
except OSError:
continue
visualize(scores, basename, div=div)
def get_conn_string_from_args() -> str:
parser = ArgParser()
parser.add(
"--server",
action="store",
default="localhost",
help="Database server name or IP address",
)
parser.add(
"--port", action="store", default="1433", help="Database server port number"
)
parser.add(
"--dbname",
action="store",
default="test_analytics_middle_tier_engage",
help="Name of the test database",
)
parser.add(
"--useintegratedsecurity",
action="store",
default=True,
help="Use Integrated Security for the database connection",
)
parser.add(
"--username",
action="store",
help="Database username when not using integrated security",
)
parser.add(
"--password",
action="store",
help="Database user password, when not using integrated security",
)
parsed = parser.parse_args(sys.argv[1:])
server = parsed.server
port = parsed.port or "1433"
db_name = parsed.dbname
username = parsed.username
password = parsed.password
integrated = parsed.useintegratedsecurity
if integrated:
return f"mssql+pyodbc://{server},{port}/{db_name}?driver=ODBC+Driver+17+for+SQL+Server&Trusted_Connection=yes"
else:
return f"mssql+pyodbc://{username}:{password}@{server},{port}/{db_name}?driver=ODBC+Driver+17+for+SQL+Server"
def initialize_arguments(p: configargparse.ArgParser):
p.add('--data', help='Data directory', type=str)
p.add('--sentence_index',
help='Column index of sentences in data file',
type=int)
p.add('--batch_size',
help='Batch size for training multi-label document classifier',
type=int)
p.add('--max_len', help='Maximum sequence length', type=int)
p.add('--epochs', help='Epochs to train', type=int)
#Optimizer arguments
p.add('--learning_rate', help='Optimizer step size', type=float)
#p.add('--weight_decay', help='Adam regularization', type=float)
return p.parse_args()
def get_fetcher_job_args(args, env_vars):
parser = ArgParser(
description="Downloads the dataset from http/ftp/s3 to internal s3")
parser.add_argument("--src", required=True, help="Source", default=None)
parser.add_argument("--dst",
metavar="dst",
required=True,
help="Destination",
default=None)
parser.add_argument("--md5", help="MD5 hash", default=None)
parser.add_argument("--zk-node-path",
help="Zookeeper node to update",
default=None)
parser.add_argument("--zookeeper-ensemble-hosts",
env_var="ZOOKEEPER_ENSEMBLE_HOSTS",
default="localhost:2181")
parser.add_argument("--logging-level",
env_var="LOGGING_LEVEL",
default="INFO")
parser.add_argument("--retry-max-attempts",
env_var="RETRY_MAX_ATTEMPTS",
type=int,
default=MAX_ATTEMPTS)
parser.add_argument("--retry-exp-max",
env_var="RETRY_EXP_MAX",
type=int,
default=MAX_WAIT_MS)
parser.add_argument("--retry-exp-multiplier",
env_var="RETRY_EXP_MULTIPLIER",
type=int,
default=EXP_MULTIPLIER_MS)
parser.add_argument("--tmp-dir", env_var="TMP_DIR", type=str)
parsed_args = parser.parse_args(args, env_vars=env_vars)
return FetcherJobConfig(
src=parsed_args.src,
dst=parsed_args.dst,
md5=parsed_args.md5,
zk_node_path=parsed_args.zk_node_path,
zookeeper_ensemble_hosts=parsed_args.zookeeper_ensemble_hosts,
logging_level=parsed_args.logging_level,
retry=RetryConfig(
exp_max=parsed_args.retry_exp_max,
exp_multiplier=parsed_args.retry_exp_multiplier,
max_attempts=parsed_args.retry_max_attempts,
),
tmp_dir=parsed_args.tmp_dir,
)
def __parse_args(self):
arg_parser = ArgParser()
subparsers = arg_parser.add_subparsers(
title="commands", dest="command"
)
self.__add_global_args(arg_parser)
for command_name, command in self.__commands.items():
subparser = subparsers.add_parser(command_name)
self.__add_global_args(subparser)
command.add_arguments(subparser, self.__add_global_args)
parsed_args = arg_parser.parse_args()
if parsed_args.command is None:
arg_parser.print_usage()
sys.exit(1)
return parsed_args
def main():
argparser = ArgParser(
description=
"Visualize scores of a model over the dev set, saving to .png file.")
argparser.add_argument("models",
nargs="+",
help="model file basename(s) to load")
args = argparser.parse_args()
for pattern in args.models:
for filename in glob(pattern) or [pattern]:
basename, _ = os.path.splitext(filename)
for div in "dev", "test":
try:
scores = load_scores(basename, div=div)
except OSError:
continue
visualize(scores, basename, div=div)
def __init__(self,chosen_dir=None):
#CMD arguments and configfile
if sys.platform == 'win32':
self.shell=True
locs = [os.path.join(sys.path[0],'phpar2.exe'),
'phpar2.exe',
os.path.join(sys.path[0],'par2.exe'),
'par2.exe',
]
par_cmd = 'par2'
for p in locs:
if os.path.isfile(p):
par_cmd = p
break
else:
self.shell=False
par_cmd = 'par2'
if chosen_dir == None:
parser = ArgParser(default_config_files=['par2deep.ini', '~/.par2deep'])
else:
parser = ArgParser(default_config_files=[os.path.join(chosen_dir,'par2deep.ini'), '~/.par2deep'])
parser.add_argument("-q", "--quiet", action='store_true', help="Don't asks questions, go with all defaults, including repairing and deleting files (default off).")
parser.add_argument("-over", "--overwrite", action='store_true', help="Overwrite existing par2 files (default off).")
parser.add_argument("-novfy", "--noverify", action='store_true', help="Do not verify existing files (default off).")
parser.add_argument("-keep", "--keep_old", action='store_true', help="Keep unused par2 files and old par2 repair files (.1,.2 and so on).")
parser.add_argument("-ex", "--excludes", action="append", type=str, default=[], help="Optionally excludes directories ('root' is files in the root of -dir).")
parser.add_argument("-exex", "--extexcludes", action="append", type=str, default=[], help="Optionally excludes file extensions.")
parser.add_argument("-dir", "--directory", type=str, default=os.getcwd(), help="Path to operate on (default is current directory).")
parser.add_argument("-pc", "--percentage", type=int, default=5, help="Set the parity percentage (default 5%%).")
parser.add_argument("-pcmd", "--par_cmd", type=str, default=par_cmd, help="Set path to alternative par2 command (default \"par2\").")
#lets get a nice dict of all o' that.
args = {k:v for k,v in vars(parser.parse_args()).items() if v is not None}
self.args = args
#add number of files
args["nr_parfiles"] = str(1) #number of parity files
#set that shit
for k,v in self.args.items():
setattr(self, k, v)
return
def parse_config() -> Namespace:
parser = ArgParser(config_file_parser_class=YAMLConfigFileParser,
default_config_files=["config.yml"])
parser.add('-c',
'--config',
is_config_file=True,
help='notification server config in yaml format (.yml)')
parser.add('-p',
'--port',
type=int,
required=True,
help='notification server port')
parser.add('-s',
'--secrets-file',
type=str,
required=True,
help='secrets file path')
return parser.parse_args()
def init_args(args):
parser = ArgParser()
parser.add_argument(
'--registry',
help=
'Specify the URL of your docker registry (use protocol prefix like http or https)',
required=True,
env_var='DOCKER_REGISTRY')
parser.add_argument('--username',
help='Specify Username',
required=False,
env_var='DOCKER_USERNAME')
parser.add_argument('--password',
help='Specify Password',
required=False,
env_var='DOCKER_PASSWORD')
parser.add_argument('--cacert',
help='Path to your custom root CA',
required=False,
env_var='DOCKER_CACERT')
parser.add_argument('--redis',
help='Hostname of your Redis instance',
required=False,
env_var='REDIS_HOST')
parser.add_argument('--listen',
help='Hostname of your Redis instance',
required=False,
env_var='LISTEN',
default="0.0.0.0")
parser.add_argument('--port',
help='Hostname of your Redis instance',
required=False,
env_var='PORT',
default="8000")
parser.add_argument('--cli',
help='Flag for a one time analysis instead of you',
required=False,
action='store_true')
parser.add_argument('--debug',
help='Start Sanic in debug mode',
required=False,
action='store_true')
return parser.parse_args(args)
def parse():
configpath = resource_filename(__name__, '../data/config.ini')
p = ArgParser(default_config_files=[configpath])
# GENERAL
general = p.add_argument_group('General')
general.add('-c', '--config', is_config_file=True, help='config file path')
general.add('-v', action='store_true', help='verbose')
general.add('-m', '--mode', help='program mode')
general.add('--multiprocessing', type=string2bool, nargs='?', const=True, default=False)
general.add('--cores', type=int)
# CLUSTERS
clusters = p.add_argument_group('Clusters')
clusters.add('--add_threshold', type=float)
clusters.add('--merge_threshold', type=float)
clusters.add('--mature_threshold', type=int)
clusters.add('--merge_frequency', type=int)
clusters.add('--remove_frequency', type=int)
clusters.add('--summary_frequency', type=int)
# EMBEDDING MODEL
model = p.add_argument_group('Embedding Model')
model.add('--type', type=str)
model.add('--source', type=str)
# DATASETS
datasets = p.add_argument_group('Datasets')
datasets.add('--background', type=str)
datasets.add('--background_weight', type=float)
datasets.add('--filter', type=string2bool, nargs='?', const=True, default=False)
datasets.add('--merge', type=string2bool, nargs='?', const=True, default=False)
# p.add('-i', '--input', help='input file name')
# p.add('-o', '--output', help='output file name')
# make a setup function to download nltk resources:
# stopwords, punkt, wordnet
return p.parse_args()
def main(graphRadius: int, branch: int, tour: bool, debug: bool, drawType: str,
pixelRadius: int, nextPieceDict: str):
if tour:
Tour(rad=graphRadius,
branch=branch,
drawType=DrawType[drawType],
debug=debug,
nxt=json.loads(nextPieceDict)).Build()
if __name__ == "__main__":
p = ArgParser(default_config_files=["graph.conf"])
p.add("--branch",
type=int,
help="in tour graph: branch factor for each hop.")
p.add("--tour", type=bool, help="whether to make a tour graph")
p.add("--debug", type=bool, help="prints debugging info")
p.add("--graphRadius",
type=int,
help="max abs(x) or abs(y) coord for graph.")
p.add("--drawType",
type=str,
help="which DrawType to use for rendoring board")
p.add("--pixelRadius",
type=int,
help="for .bmp renders, radius for each piece's color")
p.add("--nextPieceDict",
type=str,
help="defines piece ordering for a Tour")
main(**vars(p.parse_args()))
def main():
p = ArgParser()
p.add("-c", "--my-config", is_config_file=True, help="config file path")
p.add(
"-o",
"--out",
help="output directory, '-' for stdin",
type=DirectoryType(),
required=True,
)
p.add(
"-a",
"--as_of",
default=0,
help="number of days in the past to project from",
type=int,
)
p.add("-y", "--y_max", help="max y-scale for the census graph", type=int)
p.add(
"-d",
"--n_days",
help="make a census/admits plot out to n_days",
type=int,
action="append",
)
p.add("-P", "--prefix", help="prefix for filenames")
p.add(
"-pp",
"--plot_pairs",
action="store_true",
help="Plot posterior samples in a pair-plot grid",
)
p.add(
"-pc",
"--plot_capacity",
action="store_true",
help="plot capacity as a horizontal line",
)
options = p.parse_args()
prefix = ""
if options.prefix is not None:
prefix = f"{options.prefix}_"
n_days = [30, 90, 180]
if options.n_days:
n_days = options.n_days
dir = options.out
print(f"Output directory: {dir}")
paramdir = path.join(dir, "parameters")
outdir = path.join(dir, "output")
figdir = path.join(dir, "figures")
census_ts, params, args = read_inputs(paramdir)
first_day = census_ts["date"].values[0]
# TODO: This needs to be configurable based on the time period specificed
as_of_days_ago = args["as_of"]
nobs = census_ts.shape[0] - as_of_days_ago
# define capacity
vent_capacity, hosp_capacity = None, None
if options.plot_capacity:
vent_capacity = float(params.base.loc[params.param == "vent_capacity"])
hosp_capacity = float(params.base.loc[params.param == "hosp_capacity"])
# Chains
df = pd.read_json(
path.join(f"{outdir}", "chains.json.bz2"), orient="records", lines=True
)
print(f"READ chains file: {df.shape[0]} total iterations")
# remove burn-in
# TODO: Make 1000 configurable
df = df.loc[(df.iter > 1000)]
qlist = []
for day in range(census_ts.shape[0]):
ldist = logistic(
df.logistic_L, df.logistic_k, df.logistic_x0 - df.offset.astype(int), day
)
qlist.append(np.quantile(ldist, [0.05, 0.5, 0.95]))
# logistic SD plot
qmat = np.vstack(qlist)
fig = plt.figure()
plt.plot(list(range(census_ts.shape[0])), 1 - qmat[:, 1])
plt.fill_between(
x=list(range(census_ts.shape[0])),
y1=1 - qmat[:, 0],
y2=1 - qmat[:, 2],
alpha=0.3,
lw=2,
edgecolor="k",
)
plt.ylabel(f"Relative (effective) social contact")
plt.xlabel(f"Days since {first_day}")
plt.ylim(0, 1)
fig.savefig(path.join(f"{figdir}", f"{prefix}effective_soc_dist.pdf"))
for howfar in n_days:
plt_predictive(
df,
first_day,
census_ts,
figdir,
as_of_days_ago,
howfar=howfar,
prefix=prefix,
y_max=options.y_max,
hosp_capacity=hosp_capacity,
vent_capacity=vent_capacity,
)
mk_projection_tables(df, first_day, outdir)
toplot = df[
[
"beta",
"hosp_prop",
"ICU_prop",
"vent_prop",
"hosp_LOS",
"ICU_LOS",
"vent_LOS",
"incubation_days",
"recovery_days",
"logistic_k",
"logistic_x0",
"logistic_L",
"nu",
]
]
pspace = np.linspace(0.001, 0.999, 1000)
fig, ax = plt.subplots(figsize=(8, 40), ncols=1, nrows=len(toplot.columns))
for i in range(len(toplot.columns)):
cname = toplot.columns[i]
if params.loc[params.param == cname, "distribution"].iloc[0] == "gamma":
x = sps.gamma.ppf(
pspace,
params.loc[params.param == cname, "p1"],
0,
params.loc[params.param == cname, "p2"],
)
y = sps.gamma.pdf(
x,
params.loc[params.param == cname, "p1"],
0,
params.loc[params.param == cname, "p2"],
)
elif params.loc[params.param == cname, "distribution"].iloc[0] == "beta":
x = sps.beta.ppf(
pspace,
params.loc[params.param == cname, "p1"],
params.loc[params.param == cname, "p2"],
)
y = sps.beta.pdf(
x,
params.loc[params.param == cname, "p1"],
params.loc[params.param == cname, "p2"],
)
ax[i].plot(x, y, label="prior")
ax[i].hist(toplot[cname], density=True, label="posterior", bins=30)
ax[i].set_xlabel(params.loc[params.param == cname, "description"].iloc[0])
ax[i].legend()
plt.tight_layout()
fig.savefig(path.join(f"{figdir}", f"{prefix}marginal_posteriors_v2.pdf"))
if options.plot_pairs:
# Make a pair plot for diagnosing posterior dependence
plt_pairplot_posteriors(toplot, figdir, prefix=prefix)
def parse(parser: Parser, args: List[str]) -> Namespace:
# TODO: accepting a comma-separated list might allow more flexibility
default_config_file = os.getenv("PYDPIPER_CONFIG_FILE")
if default_config_file is not None:
try:
with open(default_config_file) as _:
pass
except:
warnings.warn(f"PYDPIPER_CONFIG_FILE is set to '{default_config_file}', which can't be opened.")
config_files = [default_config_file] if default_config_file else []
# First, build a parser that's aware of all options
# (will be used for help/version/error messages).
# This must be tried _before_ the partial parsing attempts
# in order to get correct help/version messages.
main_parser = ArgParser(default_config_files=config_files)
# TODO: abstract out the recursive travels in go_1 and go_2 into a `walk` function
def go_1(p, current_prefix):
if isinstance(p, BaseParser):
g = main_parser.add_argument_group(p.group_name)
for a in p.argparser._actions:
new_a = copy.copy(a)
ss = copy.deepcopy(new_a.option_strings)
for ix, s in enumerate(new_a.option_strings):
if s.startswith("--"):
ss[ix] = "-" + current_prefix + "-" + s[2:] # "" was "-"
else:
raise NotImplementedError(
"sorry, I only understand flags starting with `--` at the moment, but got %s" % s)
new_a.option_strings = ss
g._add_action(new_a)
elif isinstance(p, CompoundParser):
for q in p.parsers:
go_1(q.parser, current_prefix + (('-' + q.prefix) if q.prefix is not None else ''))
else:
raise TypeError(
"parser %s wasn't a %s (%s or %s) but a %s" % (p, Parser, BaseParser, CompoundParser, p.__class__))
go_1(parser, "")
# Use this parser to exit with a helpful message if parse fails or --help/--version specified:
main_parser.parse_args(args)
# Now, use parse_known_args for each parser in the tree of parsers to fill the appropriate namespace object ...
def go_2(p, current_prefix, current_ns):
if isinstance(p, BaseParser):
new_p = ArgParser(default_config_files=config_files)
for a in p.argparser._actions:
new_a = copy.copy(a)
ss = copy.deepcopy(new_a.option_strings)
for ix, s in enumerate(new_a.option_strings):
if s.startswith("--"):
ss[ix] = "-" + current_prefix + "-" + s[2:]
else:
raise NotImplementedError
new_a.option_strings = ss
new_p._add_action(new_a)
_used_args, _rest = new_p.parse_known_args(args, namespace=current_ns)
# add a "_flags" field to each object so we know what flags caused a certain option to be set:
# (however, note that post-parsing we may munge around ...)
flags_dict = defaultdict(set)
for action in new_p._actions:
for opt in action.option_strings:
flags_dict[action.dest].add(opt)
current_ns.flags_ = Namespace(**flags_dict)
# TODO: could continue parsing from `_rest` instead of original `args`
elif isinstance(p, CompoundParser):
current_ns.flags_ = set() # could also check for the CompoundParser case and not set flags there,
# since there will never be any
for q in p.parsers:
ns = Namespace()
if q.namespace in current_ns.__dict__:
raise ValueError("Namespace field '%s' already in use" % q.namespace)
# TODO could also allow, say, a None
else:
# gross but how to write n-ary identity fn that behaves sensibly on single arg??
current_ns.__dict__[q.namespace] = ns
# FIXME this casting doesn't work for configurations with positional arguments,
# which aren't unpacked correctly -- better to use a namedtuple
# (making all arguments keyword-only also works, but then you have to supply
# often meaningless defaults in the __init__)
go_2(q.parser, current_prefix=current_prefix + (('-' + q.prefix) if q.prefix is not None else ''),
current_ns=ns)
# If a cast function is provided, apply it to the namespace, possibly doing dynamic type checking
# and also allowing the checker to provide hinting for the types of the fields
flags = ns.flags_
del ns.flags_
fixed = (q.cast(current_ns.__dict__[q.namespace]) #(q.cast(**vars(current_ns.__dict__[q.namespace]))
if q.cast else current_ns.__dict__[q.namespace])
if isinstance(fixed, tuple):
fixed = fixed.replace(flags_=flags)
elif isinstance(fixed, Namespace):
setattr(fixed, "flags_", flags)
else:
raise ValueError("currently only Namespace and NamedTuple objects are supported return types from "
"parsing; got %s (a %s)" % (fixed, type(fixed)))
current_ns.__dict__[q.namespace] = fixed
# TODO current_ns or current_namespace or ns or namespace?
else:
raise TypeError("parser %s wasn't a %s (%s or %s) but a %s" %
(p, Parser, BaseParser, CompoundParser, p.__class__))
main_ns = Namespace()
go_2(parser, current_prefix="", current_ns=main_ns)
return main_ns
def setup_DA(conf_exp=None):
p = ArgParser()
p.add_argument('-c',
'--conf-exp',
is_config_file=True,
help='config file path',
default=conf_exp)
p.add_argument('--exp_id', type=str, required=True, help='experiment id')
p.add_argument('--datadir', type=str, required=True, help='path to data')
p.add_argument('--res_dir',
type=str,
required=True,
help='path to results')
p.add_argument('--K',
type=int,
required=True,
help='number of slow variables (grid cells)')
p.add_argument('--J',
type=int,
required=True,
help='number of fast variables (vertical levels)')
p.add_argument('--T',
type=int,
required=True,
help='length of simulation data (in time units [s])')
p.add_argument('--dt',
type=float,
required=True,
help='simulation step length (in time units [s])')
p.add_argument('--N_trials',
type=int,
default=1,
help='number of random starting points for solver')
p.add_argument('--back_solve_dt_fac',
type=int,
default=100,
help='step size decrease factor for backsolve')
p.add_argument('--spin_up_time',
type=float,
default=5.,
help='spin-up time for simulation in [s]')
p.add_argument('--l96_F',
type=float,
default=10.,
help='Lorenz-96 parameter F')
p.add_argument('--l96_h',
type=float,
default=1.,
help='Lorenz-96 parameter h')
p.add_argument('--l96_b',
type=float,
default=10.,
help='Lorenz-96 parameter b')
p.add_argument('--l96_c',
type=float,
default=10.,
help='Lorenz-96 parameter c')
p.add_argument('--obs_operator',
type=str,
required=True,
help='string for observation operator class')
p.add_argument('--obs_operator_r',
type=float,
default=0.,
help='fraction of unobserved state entries')
p.add_argument('--obs_operator_sig2',
type=float,
default=1.0,
help='variance of additive observation noise')
p.add_argument('--obs_operator_frq',
type=int,
default=4,
help='cycle length for rotating observation operator')
p.add_argument('--T_rollout',
type=int,
required=True,
help='maximum length of rollout')
p.add_argument('--T_pred', type=int, required=True, help='prediction time')
p.add_argument(
'--n_chunks',
type=int,
required=True,
help='number of chunks for rollout (e.g. one obs per chunk)')
p.add_argument('--n_chunks_recursive',
type=int,
required=True,
help='finer-scale chunking, e.g. for backsolve')
p.add_argument('--n_starts',
type=int,
required=True,
help='number of rollout trials to include')
p.add_argument('--model_exp_id',
type=int,
required=True,
help='exp_id for emulator-training experiment')
p.add_argument('--model_forwarder',
type=str,
default='rk4_default',
help='string for model forwarder (e.g. RK4)')
p.add_argument('--optimizer',
type=str,
default='LBFGS',
help='string specifying numerical optimizer')
p.add_argument('--n_steps',
type=int,
required=True,
help='number of optimization steps')
p.add_argument('--lr', type=float, default=1e0, help='learning rate')
p.add_argument('--max_iter',
type=int,
default=1000,
help='number of maximum iterations per step')
p.add_argument('--max_eval',
type=int,
default=-1,
help='number of maximum evaluations per step')
p.add_argument('--tolerance_grad',
type=float,
default=1e-12,
help='convergence criterion for gradient norm')
p.add_argument('--tolerance_change',
type=float,
default=1e-12,
help='convergence criterion for function change')
p.add_argument('--history_size',
type=int,
default=10,
help='history size for L-BFGS')
p.add_argument('--if_LBFGS_chunks',
type=int,
default=0,
help='if to include chunks optimization scheme')
p.add_argument('--if_LBFGS_full_chunks',
type=int,
default=0,
help='if to include full chunks optimization')
p.add_argument('--if_backsolve',
type=int,
default=0,
help='if to include backsolve optimization scheme')
p.add_argument('--if_LBFGS_full_backsolve',
type=int,
default=0,
help='if to include full backsolve optimization')
p.add_argument('--if_LBFGS_full_persistence',
type=int,
default=0,
help='if to include persistence optimization')
p.add_argument('--if_LBFGS_recurse_chunks',
type=int,
default=0,
help='if to include recursive optimization scheme')
args = p.parse_args() if conf_exp is None else p.parse_args(args=[])
return vars(args)
def _process_args(self):
flags = ArgParser(prog='repoman',
add_config_file_help=True,
ignore_unknown_config_file_keys=True,
default_config_files=self.config_files)
flags.add('--config-file', required=False, is_config_file=True,
env_var='REPOMAN_CONFIG_FILE',
help='override config file path')
# global flags
flags.add('--simpledb-domain', action='store', required=True,
env_var='REPOMAN_SIMPLEDB_DOMAIN')
flags.add('--s3-bucket', action='store', required=True,
env_var='REPOMAN_S3_BUCKET')
flags.add('--aws-profile', action='store', required=False, default='',
env_var='REPOMAN_AWS_CREDENTIAL_PROFILE',
help='Use the specified profile in ~/.aws/credentials')
flags.add('--region', action='store', required=False,
default=None, help='AWS region to connect to')
flags.add('--aws-role', action='store', required=False, default='',
env_var='REPOMAN_AWS_ROLE',
help='Full ARN of IAM role to assume before calling any '
'other AWS APIs')
flags.add('--log-config', action='store', required=False, default='',
env_var='REPOMAN_LOG_CONFIG',
help='path to a JSON file with a python log configuration ')
flags.add('--skip-checkup', action='store_true',
required=False, default=False,
help='do not run system health checkup on every action')
flags.add('--debug', action='store_true', required=False,
default=False, help='debug logging')
flags.add('--gpg-home',
required=False, env_var='REPOMAN_GPG_HOME',
default='~/.gnupg',
help='set path to gpg keyring')
flags.add('--gpg-signer', action='append',
required=False, help='gpg identity to sign as')
flags.add('--gpg-pinentry-path', action='store',
default='/usr/bin/pinentry',
required=False, help='path to gpg pinentry program')
flags.add('--gpg-passphrase', action='append',
required=False,
help='passphrase for gpg secret key for signing '
'(if multiple, must be in same order as --gpg-signer)')
flags.add('--auto-purge', action='store', default=0,
type=int, required=False,
help='automatically purge packages older than the '
'last N revisions when adding or copying')
# subparsers for commands
commands = flags.add_subparsers(dest='command')
# singelton commands
commands.add_parser('checkup', help='check that all systems are go')
commands.add_parser('backup',
help='dump the simpledb state to a JSON file')
# restore command
restore_flags = commands.add_parser(
'restore', help='restore simpledb state from a JSON file')
restore_flags.add(
'filename', nargs=1, help='path to backup file')
# commands that take flags
setup_flags = commands.add_parser(
'setup',
help='do initial system configuration: create simpledb domain '
'and s3 bucket, specify at least one each architecture, '
'distribution and component to publish.'
)
repo_flags = commands.add_parser(
'repo', help='repo management commands')
add_flags = commands.add_parser(
'add', help='add package files to repo')
cp_flags = commands.add_parser(
'cp', help='move packages between components and distributions')
rm_flags = commands.add_parser(
'rm', help='remove specific packages from repo')
publish_flags = commands.add_parser(
'publish', help='publish the repository to s3')
query_flags = commands.add_parser(
'query', help='query the repository')
# command flags
# query
query_flags.add('-a', '--architecture',
action='append', required=False,
help='narrow query by architecture(s)')
query_flags.add('-d', '--distribution',
action='append', required=False,
help='narrow query by distribution(s)')
query_flags.add('-c', '--component',
action='append', required=False,
help='narrow query by component(s)')
query_flags.add('-p', '--package',
action='append', required=False,
help='narrow query by package name(s)')
query_flags.add('-w', '--wildcard', action='store_true', default=False,
help='match package names to left of --package flag')
query_flags.add('-H', '--query-hidden', action='store_true',
default=False,
help='include packages "hidden" by the removal of '
'their distribution/component/architecture')
query_flags.add('-f', '--format', action='store',
dest='outputfmt', default='simple',
choices=('json', 'jsonc', 'packages', 'simple',
'plain', 'grid', 'fancy_grid', 'pipe',
'orgtbl', 'jira', 'psql', 'rst', 'mediawiki',
'moinmoin', 'html', 'latex', 'latex_booktabs',
'textile'),
help='select output format for querys & rm/cp prompts')
query_latest = query_flags.add_mutually_exclusive_group()
query_latest.add('-v', '--version', action='append',
help='only return packages matching these versions')
query_latest.add('-l', '--latest', action='store_const',
dest='latest_versions', const=1,
help='only return the most recent package version '
'(equivalent to `--recent 1`)')
query_latest.add('-r', '--recent', action='store', default=0,
type=int, dest='latest_versions',
help='only return the N most recent package versions')
# setup
setup_flags.add('-a', '--architecture',
action='append', required=True,
help='specify at least one architecture')
setup_flags.add('-d', '--distribution',
action='append', required=True,
help='specify at least one distribution')
setup_flags.add('-c', '--component',
action='append', required=True,
help='specify at least one component')
setup_flags.add('--s3-acl', action='store',
default='private', required=False,
choices=S3_BUCKET_ACLS,
help='set a canned ACL for the S3 bucket '
'(default is private)')
setup_flags.add('--s3-region', action='store', required=False,
help='set region for s3 bucket '
'(default is us-east-1 AKA US/Standard)')
setup_flags.add('--sns-topic', action='store', required=False,
help='AWS SNS topic name for logging')
setup_flags.add('--origin', action='store', required=False,
help='origin string for repository')
setup_flags.add('--label', action='store', required=False,
help='label string for repository')
setup_flags.add('--enable-website', action='store_true',
required=False, default=False,
help='configure public website hosting for '
'the S3 bucket. Implies --s3-acl=public-read')
# repo management operations
repo_commands = repo_flags.add_subparsers(dest='repo_command')
repo_add_architecture_flags = repo_commands.add_parser(
'add-architecture', help='add a architecture to repo')
repo_add_architecture_flags.add(
'architecture_names', nargs='+', help='architecture to add')
repo_add_architecture_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
raa_confirm = \
repo_add_architecture_flags.add_mutually_exclusive_group()
raa_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
raa_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_rm_architecture_flags = repo_commands.add_parser(
'rm-architecture', help='remove a architecture from repo')
repo_rm_architecture_flags.add(
'architecture_names', nargs='+', help='architecture to remove')
repo_rm_architecture_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rra_confirm = repo_rm_architecture_flags.add_mutually_exclusive_group()
rra_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rra_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_add_distribution_flags = repo_commands.add_parser(
'add-distribution', help='add a distribution to repo')
repo_add_distribution_flags.add(
'distribution_names', nargs='+', help='distribution to add')
repo_add_distribution_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rad_confirm = \
repo_add_distribution_flags.add_mutually_exclusive_group()
rad_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rad_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_rm_distribution_flags = repo_commands.add_parser(
'rm-distribution', help='remove a distribution from repo')
repo_rm_distribution_flags.add(
'distribution_names', nargs='+', help='distribution to remove')
repo_rm_distribution_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rrd_confirm = repo_rm_distribution_flags.add_mutually_exclusive_group()
rrd_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rrd_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_add_component_flags = repo_commands.add_parser(
'add-component', help='add a component to repo')
repo_add_component_flags.add(
'component_names', nargs='+', help='component to add')
repo_add_component_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rac_confirm = repo_add_component_flags.add_mutually_exclusive_group()
rac_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rac_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_rm_component_flags = repo_commands.add_parser(
'rm-component', help='remove a component from repo')
repo_rm_component_flags.add(
'component_names', nargs='+', help='component to remove')
repo_rm_component_flags.add(
'--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rrc_confirm = repo_rm_component_flags.add_mutually_exclusive_group()
rrc_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rrc_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_add_topic_flags = repo_commands.add_parser(
'add-topic', help='send notifications to an SNS topic')
repo_add_topic_flags.add('topic_name', nargs=1, action='store',
help='SNS topic to configure for logging')
rat_confirm = repo_add_topic_flags.add_mutually_exclusive_group()
rat_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rat_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_rm_topic_flags = repo_commands.add_parser(
'rm-topic', help='remove SNS topic logging')
rrt_confirm = repo_rm_topic_flags.add_mutually_exclusive_group()
rrt_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rrt_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_commands.add_parser('show-config',
help='show current repo configuration')
repo_add_origin_flags = repo_commands.add_parser(
'add-origin', help='set the repository origin string')
repo_add_origin_flags.add('origin', nargs=1, action='store',
help='origin string')
rao_confirm = repo_add_origin_flags.add_mutually_exclusive_group()
rao_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rao_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_add_label_flags = repo_commands.add_parser(
'add-label', help='set the repository label string')
repo_add_label_flags.add('label', nargs=1, action='store',
help='label string')
ral_confirm = repo_add_label_flags.add_mutually_exclusive_group()
ral_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
ral_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
repo_commands.add_parser('show-config',
help='show current repo configuration')
# add packages
add_flags.add('-d', '--distribution', action='append', required=True,
help='add to specified distribution')
add_flags.add('-c', '--component', action='append', required=True,
help='add to specified component')
add_flags.add('--overwrite',
action='store_true', required=False, default=False,
help='re-upload packages even if they already exist '
'in the repository')
add_flags.add('--publish',
action='store_true', required=False, default=False,
help='publish the repo to s3 after adding packages')
add_flags.add('files', nargs='+', help='debian package files to add')
# copy
cp_flags.add('--src-distribution', action='store', required=True,
help='specify one or more distributions to copy from')
cp_flags.add('--dst-distribution', action='store',
help='specify one or more distributions to copy to')
cp_flags.add('--src-component', action='store', required=True,
help='specify one or more components to copy from')
cp_flags.add('--dst-component', action='store',
help='specify one or more components to copy to')
cp_flags.add('-a', '--architecture', action='append', required=False,
help='limit to specified architectures')
cp_flags.add('-p', '--package', action='append',
help='specify one or more package names to act on')
cp_flags.add('--overwrite',
action='store_true', required=False, default=False,
help='re-upload packages even if they already exist '
'in the repository -- this only applies to cross-'
'distribution copies')
cp_flags.add('--promote',
action='store_true', required=False, default=False,
help='only copy files where the latest source version '
'is more recent than the latest destination version ')
cp_flags.add('-w', '--wildcard', action='store_true', default=False,
help='match package names to left of --package flag')
cp_latest = cp_flags.add_mutually_exclusive_group()
cp_latest.add('-v', '--version', action='append',
help='only copy packages matching these versions')
cp_latest.add('-l', '--latest', action='store_const',
dest='latest_versions', const=1,
help='only copy the most recent package version '
'(equivalent to `--recent 1`)')
cp_latest.add('-r', '--recent', action='store', default=0,
type=int, dest='latest_versions',
help='only copy the N most recent package versions')
cp_flags.add('--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
cp_confirm = cp_flags.add_mutually_exclusive_group()
cp_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
cp_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
# remove
rm_flags.add('-a', '--architecture', action='append', required=False,
help='limit to specified architectures')
rm_flags.add('-d', '--distribution', action='append', required=False,
help='limit to specified distributions')
rm_flags.add('-c', '--component', action='append', required=False,
help='limit to specified distributions')
rm_flags.add('-p', '--package', action='append', required=False,
help='limit to specified package names')
rm_flags.add('--remove-from-s3', action='store_true',
default=False, required=False,
help='remove package files from s3')
rm_flags.add('--publish', action='store_true', required=False,
default=False, help='publish the repo to s3')
rm_flags.add('-w', '--wildcard', action='store_true', default=False,
help='match package names to left of --package flag')
rm_flags.add('-H', '--rm-hidden', action='store_true',
default=False,
help='include packages "hidden" by the removal of '
'their distribution/component/architecture')
rm_flags.add('--i-fear-no-evil', action='store_true',
default=False, required=False,
help='skip confirmation step for scary actions')
rm_flags.add('-f', '--format', action='store',
dest='outputfmt', default='simple',
choices=('json', 'jsonc', 'simple', 'plain', 'grid',
'fancy_grid', 'pipe', 'orgtbl', 'jira',
'psql', 'rst', 'mediawiki', 'moinmoin',
'html', 'latex', 'latex_booktabs', 'textile'),
help='select output format for querys & rm/cp prompts')
rm_latest = rm_flags.add_mutually_exclusive_group()
rm_latest.add('-v', '--version', action='append',
help='only delete packages matching these versions')
rm_latest.add('-l', '--exclude-latest', action='store_const',
dest='latest_versions', const=1,
help='only delete the most recent package version '
'(equivalent to `--recent 1`)')
rm_latest.add('-r', '--exclude-recent', action='store', default=0,
type=int, dest='latest_versions',
help='only delete the N most recent package versions')
rm_confirm = rm_flags.add_mutually_exclusive_group()
rm_confirm.add('--confirm', action='store_true', dest='confirm',
required=False, default=True,
help='confirm any mutating actions')
rm_confirm.add('-y', '--no-confirm', action='store_false',
dest='confirm', required=False, default=False,
help='do not prompt for confirmation')
# publish to s3
publish_flags.add('-d', '--distribution', action='append',
required=False,
help='limit to specified distributions '
'(default is all)')
config = flags.parse_args(self.argv)
return config
def program_options():
"""Create argument parser for the CLI.
"""
parser = ArgParser()
dp_group = parser.add_argument_group(title="Required DP Parameters")
io_group = parser.add_argument_group(
title="Required Train Data Parameters")
# File IO
io_group.add(
"--train-normal",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
# DP Req. Options
dp_group.add(
"--epsilon",
nargs="+",
help="Epsilon value(s) for differentially-private training. "\
"One or many epsilon values can be specified. If multiple epsilons are "\
"specified, then independent experiments will be run for each specified "\
"epislon value. The results of each of these runs will be stored in "\
"separate, named result files. "\
"Epsilons can be specified as decimal values. Some examples of valid "\
"epsilon arguments are "\
"`--epsilon 3`, "\
"`--epsilon 5.32341`, "\
"`--epsilon 3 3.5 4 4.5 20`.",
type=float,
required=True
)
dp_group.add(
"--delta",
nargs="+",
help= "Delta value(s) for differentially-private training. "\
"One or many delta values can be specified. If multiple deltas are"\
"specified, then independent experiments will be run for each delta"\
"value in combination with each epsilon value."\
"The reuslts of these runs are stored in separate, named result files."\
"To use (eps)-DP for privacy calculations, pass use the option\n"\
"`--delta 0`.",
type=float,
required=True
)
parser.add("--participant",
metavar="PARTICPANT",
help="Server or client.",
choices=["server", "client"],
required=True)
parser.add("--training-seed",
help="Seed used for the training.",
type=int,
default=42)
parser.add(
"--output-dir",
help="Directory to store output result files to. If the directory does not "\
"exist, it will be created.",
type=str,
default="."
)
parser.add("--host",
help="Specifies the server address.",
default="localhost",
type=str)
parser.add(
"--port",
help=
"Specifies the port through which the two workers should communicate on the host machine.",
default="8081",
type=int)
parser.add("--mode",
help="Specifies the launching mode.",
default="subprocess",
choices=["subprocess", "docker"],
type=str)
args = parser.parse_args()
# Some post processing, for lists etc.
if not isinstance(args.epsilon, list):
args.epsilon = [
args.epsilon,
]
if not isinstance(args.delta, list):
args.delta = [
args.delta,
]
# Convert all relative file paths into absolute paths
resolve_path = lambda p: str(pathlib.Path(p).resolve())
args.train_normal = resolve_path(args.train_normal)
args.train_tumor = resolve_path(args.train_tumor)
if args.output_dir is not None:
args.output_dir = resolve_path(args.output_dir)
return args
def setup_4DVar(conf_exp=None):
p = ArgParser()
p.add_argument('-c',
'--conf-exp',
is_config_file=True,
help='config file path',
default=conf_exp)
p.add_argument('--exp_id', type=str, required=True, help='experiment id')
p.add_argument('--datadir', type=str, required=True, help='path to data')
p.add_argument('--res_dir',
type=str,
required=True,
help='path to results')
p.add_argument('--T_win',
type=int,
required=True,
help='4D-Var integration window in steps')
p.add_argument(
'--T_shift',
type=int,
default=-1,
help='per-analysis shift of 4D-Var integration window in steps')
p.add_argument('--B',
type=float,
required=True,
help='scale for covariance matrix of init-state prior')
p.add_argument('--K',
type=int,
required=True,
help='number of slow variables (grid cells)')
p.add_argument('--J',
type=int,
required=True,
help='number of fast variables (vertical levels)')
p.add_argument('--T',
type=int,
required=True,
help='length of simulation data (in time units [s])')
p.add_argument('--dt',
type=float,
required=True,
help='simulation step length (in time units [s])')
p.add_argument('--N_trials',
type=int,
default=1,
help='number of random starting points for solver')
p.add_argument('--spin_up_time',
type=float,
default=5.,
help='spin-up time for simulation in [s]')
p.add_argument('--l96_F',
type=float,
default=10.,
help='Lorenz-96 parameter F')
p.add_argument('--l96_h',
type=float,
default=1.,
help='Lorenz-96 parameter h')
p.add_argument('--l96_b',
type=float,
default=10.,
help='Lorenz-96 parameter b')
p.add_argument('--l96_c',
type=float,
default=10.,
help='Lorenz-96 parameter c')
p.add_argument('--obs_operator',
type=str,
required=True,
help='string for observation operator class')
p.add_argument('--obs_operator_r',
type=float,
default=0.,
help='fraction of unobserved state entries')
p.add_argument('--obs_operator_sig2',
type=float,
default=1.0,
help='variance of additive observation noise')
p.add_argument('--obs_operator_frq',
type=int,
default=4,
help='cycle length for rotating observation operator')
p.add_argument('--model_exp_id',
type=int,
required=True,
help='exp_id for emulator-training experiment')
p.add_argument('--model_forwarder',
type=str,
default='rk4_default',
help='string for model forwarder (e.g. RK4)')
p.add_argument('--optimizer',
type=str,
default='LBFGS',
help='string specifying numerical optimizer')
p.add_argument('--n_steps',
type=int,
required=True,
help='number of optimization steps')
p.add_argument('--lr', type=float, default=1e0, help='learning rate')
p.add_argument('--max_iter',
type=int,
default=1000,
help='number of maximum iterations per step')
p.add_argument('--max_eval',
type=int,
default=-1,
help='number of maximum evaluations per step')
p.add_argument('--tolerance_grad',
type=float,
default=1e-12,
help='convergence criterion for gradient norm')
p.add_argument('--tolerance_change',
type=float,
default=1e-12,
help='convergence criterion for function change')
p.add_argument('--history_size',
type=int,
default=10,
help='history size for L-BFGS')
args = p.parse_args() if conf_exp is None else p.parse_args(args=[])
return vars(args)
def parse_main_arguments(args_in: List[str]) -> MainArguments:
"""
Configures the command-line interface.
Parameters
----------
args_in : list of str
Full argument list from the command line.
Returns
-------
arguments : MainArguments
A populated `MainArguments` object.
"""
parser = ArgParser()
parser.add( # type: ignore
"-s",
"--server",
help="Database server name or IP address",
required=True,
env_var="DB_SERVER",
)
parser.add( # type: ignore
"--port",
help="Database server port number",
type=int,
env_var="DB_PORT",
default=1433,
)
parser.add( # type: ignore
"-d",
"--dbname",
help="Name of the database with the LMS tables.",
env_var="DB_NAME",
required=True,
)
USE_INTEGRATED = "--useintegratedsecurity"
USE_INTEGRATED_SHORT = "-i"
parser.add( # type: ignore
USE_INTEGRATED_SHORT,
USE_INTEGRATED,
help="Use Integrated Security for the database connection.",
action="store_true",
)
user_name_required = (USE_INTEGRATED not in args_in
and USE_INTEGRATED_SHORT not in args_in)
# Retrieve this value because we need it in order to determine
# if username and password are required
integrated_env_var = os.getenv("USE_INTEGRATED_SECURITY")
if integrated_env_var and integrated_env_var.lower() in ("true", "yes",
"t", "y"):
user_name_required = False
parser.add( # type: ignore
"-u",
"--username",
required=user_name_required,
env_var="DB_USERNAME",
help="Database username, when not using integrated security.",
)
parser.add( # type: ignore
"-p",
"--password",
required=user_name_required,
env_var="DB_PASSWORD",
help="Database user password, when not using integrated security.",
)
parser.add( # type: ignore
"-l",
"--log-level",
required=False,
help="The log level for the tool.",
choices=LOG_LEVELS,
type=str,
default="INFO",
env_var="LOG_LEVEL",
)
parser.add( # type: ignore
"-e",
"--exceptions-report-directory",
required=False,
help="File path for optional output of a CSV exception report.",
type=str,
env_var="EXCEPTIONS_REPORT_DIRECTORY",
)
parser.add( # type: ignore
"-n",
"--encrypt",
help="Encrypt the connection to the database.",
action="store_true",
env_var="ENCRYPT_SQL_CONNECTION",
)
parser.add( # type: ignore
"-t",
"--trust-certificate",
help=
"When encrypting connections, trust the server certificate. Useful for localhost debugging with a self-signed certificate. USE WITH CAUTION.",
action="store_true",
env_var="TRUST_SERVER_CERTIFICATE",
)
parser.add( # type: ignore
"-g", # because 'e' and 'n' are already taken (；′⌒`)
"--engine",
help=
"Database engine: mssql for Microsoft SQL Server, or postgresql for PostgreSQL.",
choices=[DB_ENGINE.MSSQL, DB_ENGINE.POSTGRESQL],
default=DB_ENGINE.MSSQL,
type=str,
env_var="DB_ENGINE",
)
args_parsed = parser.parse_args(args_in)
# Need to add this back in because reading it manually earlier
# seems to cause it to be misread by the parser.
args_parsed.useintegratedsecurity = (args_parsed.useintegratedsecurity
or not user_name_required)
arguments = MainArguments(
args_parsed.log_level, args_parsed.exceptions_report_directory,
args_parsed.engine, args_parsed.server, args_parsed.dbname,
args_parsed.username, args_parsed.password, args_parsed.port,
args_parsed.encrypt, args_parsed.trust_certificate,
args_parsed.useintegratedsecurity)
return arguments
def program_options():
"""Create argument parser for the CLI.
"""
parser = ArgParser()
dp_group = parser.add_argument_group(title="Required DP Parameters")
io_group = parser.add_argument_group(
title="Required Train Data Parameters")
comm_group = parser.add_argument_group(
title="Flags for Communication (no touch)")
# File IO
io_group.add(
"--train-normal-alice",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor-alice",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
io_group.add(
"--train-normal-bob",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor-bob",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
# DP Req. Options
dp_group.add(
"--epsilon",
nargs="+",
help="Epsilon value(s) for differentially-private training. "\
"One or many epsilon values can be specified. If multiple epsilons are "\
"specified, then independent experiments will be run for each specified "\
"epislon value. The results of each of these runs will be stored in "\
"separate, named result files. "\
"Epsilons can be specified as decimal values. Some examples of valid "\
"epsilon arguments are "\
"`--epsilon 3`, "\
"`--epsilon 5.32341`, "\
"`--epsilon 3 3.5 4 4.5 20`.",
type=float,
required=True
)
dp_group.add(
"--delta",
nargs="+",
help= "Delta value(s) for differentially-private training. "\
"One or many delta values can be specified. If multiple deltas are"\
"specified, then independent experiments will be run for each delta"\
"value in combination with each epsilon value."\
"The reuslts of these runs are stored in separate, named result files."\
"To use (eps)-DP for privacy calculations, pass use the option\n"\
"`--delta 0`.",
type=float,
required=True
)
# Optional arguments
parser.add(
"--output-dir",
help="Directory to store output trained model to. If the directory does not "\
"exist, it will be created.",
type=str,
default="owkin-results"
)
# Next arguments
comm_group.add(
"--port",
help=
"Specifies the port through which the two workers should communicate on the host machine.",
default="8081",
type=int)
comm_group.add(
"--subprocess",
help="If set, the training will be performed between two subprocesses. If unset (default), "\
"the training will be performed with Docker containers.",
default=False,
action="store_true"
)
args = parser.parse_args()
# Some post processing, for lists etc.
if not isinstance(args.epsilon, list):
args.epsilon = [
args.epsilon,
]
if not isinstance(args.delta, list):
args.delta = [
args.delta,
]
return args
def parse_main_arguments(args_in: List[str]) -> MainArguments:
"""
Configures the command-line interface.
Parameters
----------
args_in : list of str
Full argument list from the command line.
Returns
-------
arguments : MainArguments
A populated `MainArguments` object.
"""
parser = ArgParser()
parser.add( # type: ignore
"-c",
"--csvpath",
help="Base path for input files.",
required=True,
env_var="CSV_PATH",
)
parser.add( # type: ignore
"-e",
"--engine",
help="Database engine.",
choices=[DbEngine.MSSQL, DbEngine.POSTGRESQL],
default=DbEngine.MSSQL,
env_var="DB_ENGINE",
)
parser.add( # type: ignore
"-s",
"--server",
help="Database server name or IP address",
required=True,
env_var="DB_SERVER",
)
parser.add( # type: ignore
"--port",
help="Database server port number",
type=int,
env_var="DB_PORT",
)
parser.add( # type: ignore
"-d",
"--dbname",
help="Name of the database with the LMS tables.",
env_var="DB_NAME",
required=True,
)
USE_INTEGRATED = "--useintegratedsecurity"
USE_INTEGRATED_SHORT = "-i"
parser.add( # type: ignore
USE_INTEGRATED_SHORT,
USE_INTEGRATED,
help="Use Integrated Security for the database connection.",
action="store_true",
)
user_name_required = (USE_INTEGRATED not in args_in
and USE_INTEGRATED_SHORT not in args_in)
# Retrieve this value because we need it in order to determine
# if username and password are required
integrated_env_var = os.getenv("USE_INTEGRATED_SECURITY")
if integrated_env_var and integrated_env_var.lower() in ("true", "yes",
"t", "y"):
user_name_required = False
parser.add( # type: ignore
"-u",
"--username",
required=user_name_required,
env_var="DB_USERNAME",
help="Database username, when not using integrated security.",
)
parser.add( # type: ignore
"-p",
"--password",
required=user_name_required,
env_var="DB_PASSWORD",
help="Database user password, when not using integrated security.",
)
parser.add( # type: ignore
"-l",
"--log-level",
required=False,
help="The log level for the tool.",
choices=LOG_LEVELS,
type=str,
default="INFO",
env_var="LOG_LEVEL",
)
parser.add( # type: ignore
"-n",
"--encrypt",
help="Encrypt the connection to the database.",
action="store_true",
env_var="ENCRYPT_SQL_CONNECTION",
)
parser.add( # type: ignore
"-t",
"--trust-certificate",
help=
"When encrypting connections, trust the server certificate. Useful for localhost debuggin with a self-signed certificate. USE WITH CAUTION.",
action="store_true",
env_var="TRUST_SERVER_CERTIFICATE",
)
args_parsed = parser.parse_args(args_in)
# Need to add this back in because reading it manually earlier
# seems to cause it to be misread by the parser.
args_parsed.useintegratedsecurity = (args_parsed.useintegratedsecurity
or not user_name_required)
arguments = MainArguments(
args_parsed.csvpath,
args_parsed.engine,
args_parsed.log_level,
args_parsed.server,
args_parsed.dbname,
args_parsed.port,
args_parsed.encrypt,
args_parsed.trust_certificate,
)
if args_parsed.useintegratedsecurity and args_parsed.engine == DbEngine.MSSQL:
arguments.build_mssql_adapter_with_integrated_security()
elif args_parsed.engine == DbEngine.MSSQL:
arguments.build_mssql_adapter(
args_parsed.username,
args_parsed.password,
)
elif args_parsed.engine == DbEngine.POSTGRESQL:
arguments.build_pgsql_adapter(
args_parsed.username,
args_parsed.password,
)
return arguments
if (all([os.path.exists(manifest) for manifest in self.manifests.values()])
and not self.overwrite):
print("Found manfiest files, skipping ingest, use --overwrite to overwrite them.")
return
for setn, manifest in self.manifests.items():
pairs = self.train_or_val_pairs(setn)
records = [(os.path.relpath(fname, self.out_dir), int(tgt))
for fname, tgt in pairs]
records.insert(0, ('@FILE', 'STRING'))
np.savetxt(manifest, records, fmt='%s\t%s')
if __name__ == "__main__":
parser = ArgParser()
parser.add_argument('--input_dir', required=True,
help='Directory to find input tars', default=None)
parser.add_argument('--out_dir', required=True,
help='Directory to write ingested files', default=None)
parser.add_argument('--target_size', type=int, default=256,
help='Size in pixels to scale shortest side DOWN to (0 means no scaling)')
parser.add_argument('--overwrite', action='store_true', default=False, help='Overwrite files')
args = parser.parse_args()
logger = logging.getLogger(__name__)
bw = IngestI1K(input_dir=args.input_dir, out_dir=args.out_dir, target_size=args.target_size,
overwrite=args.overwrite)
bw.run()
action='store_true',
help='If given, shuffle data along sample dimension.')
p.set_defaults(shuffle=False)
p.add_argument('--only_norm',
dest='only_norm',
action='store_true',
help='If given, only compute and save normalization file.')
p.set_defaults(only_norm=False)
p.add_argument('--flx_same_dt',
dest='flx_same_dt',
action='store_true',
help='If given, take surface fluxes from same time step.')
p.set_defaults(flx_same_dt=False)
p.add_argument('--norm_features',
type=str,
default=None,
help='by_var or by_lev')
p.add_argument('--norm_targets',
type=str,
default=None,
help='norm or scale')
p.add_argument('--verbose',
dest='verbose',
action='store_true',
help='If given, print debugging information.')
p.set_defaults(verbose=False)
args = p.parse_args()
main(args)
if "amr" not in keep: # Remove AMR-specific features: node label and category
delete_if_exists((model.feature_params, model.classifier.params), (NODE_LABEL_KEY, "c"))
delete_if_exists((model.classifier.labels, model.classifier.axes), {NODE_LABEL_KEY}.union(FORMATS).difference(keep))
def delete_if_exists(dicts, keys):
for d in dicts:
for key in keys:
try:
del d[key]
except KeyError:
pass
def main(args):
os.makedirs(args.out_dir, exist_ok=True)
for filename in args.models:
model = load_model(filename)
strip_multitask(model, args.keep)
model.filename = os.path.join(args.out_dir, os.path.basename(filename))
model.save()
if __name__ == "__main__":
argparser = ArgParser(description="Load TUPA model and save with just one task's features/weights.")
argparser.add_argument("models", nargs="+", help="model file basename(s) to load")
argparser.add_argument("-k", "--keep", nargs="+", choices=tuple(filter(None, FORMATS)), default=["ucca"],
help="tasks to keep features/weights for")
argparser.add_argument("-o", "--out-dir", default=".", help="directory to write modified model files to")
main(argparser.parse_args())
def _initialize_arguments(p: configargparse.ArgParser):
p.add('--model_storage_directory', help='The directory caching all model runs')
args = p.parse_args()
parser.add_argument('--input_dir',
help='Directory to find input',
default='/hdd/Dataset/Flower102')
parser.add_argument(
'--out_dir',
help='Directory to write ingested files',
default='/home/william/PyProjects/TFcodes/dataset/flower102')
parser.add_argument(
'--target_size',
type=int,
default=256,
help=
'Size in pixels to scale shortest side DOWN to (0 means no scaling)')
parser.add_argument('--ratio',
type=float,
default=0.3,
help='Percentage of dataset to be used for validation')
parser.add_argument('--skipImg',
type=bool,
default=True,
help='True to skip processing and copying images')
args = parser.parse_args()
logger = logging.getLogger(__name__)
bw = Ingest(input_dir=args.input_dir,
out_dir=args.out_dir,
target_size=args.target_size,
skipimg=args.skipImg)
bw.run()
def parse_main_arguments(args_in: List[str]) -> MainArguments:
"""
Configures the command-line interface.
Parameters
----------
args_in : list of str
Full argument list from the command line.
Returns
-------
arguments : MainArguments
A populated `MainArguments` object.
"""
parser = ArgParser()
parser.add( # type: ignore
"-k",
"--client-key",
required=True,
help="Schoology client key.",
type=str,
env_var="SCHOOLOGY_KEY",
)
parser.add( # type: ignore
"-s",
"--client-secret",
required=True,
help="Schoology client secret.",
type=str,
env_var="SCHOOLOGY_SECRET",
)
parser.add( # type: ignore
"-o",
"--output-directory",
required=False,
help="The output directory for the generated csv files.",
type=str,
default="",
env_var="OUTPUT_DIRECTORY",
)
parser.add( # type: ignore
"-l",
"--log-level",
required=False,
help="The log level for the tool.",
choices=constants.LOG_LEVELS,
type=str,
default="INFO",
env_var="LOG_LEVEL",
)
parser.add( # type: ignore
"-p",
"--page-size",
required=False,
help="Page size for the paginated requests.",
type=int,
default=200,
env_var="PAGE_SIZE",
)
parser.add( # type: ignore
"-i",
"--input-directory",
required=False,
help="Input directory for usage CSV files.",
type=str,
default=None,
env_var="SCHOOLOGY_INPUT_DIRECTORY",
)
parser.add( # type: ignore
"-d",
"--sync-database-directory",
required=False,
help="The directory for the sync database.",
type=str,
default="data",
env_var="SYNC_DATABASE_DIRECTORY",
)
parser.add( # type: ignore
"-f",
"--feature",
required=False,
help="Features to include.",
type=str,
nargs='*',
choices=constants.VALID_FEATURES,
default=[],
env_var="FEATURE",
)
args_parsed = parser.parse_args(args_in)
# Required
assert isinstance(
args_parsed.client_key, str
), "Argument `client-key` must be a string."
assert isinstance(
args_parsed.client_secret, str
), "Argument `client-secret` must be a string."
# Optional
assert isinstance(
args_parsed.output_directory, str
), "The specified `output-directory` is not valid."
assert (
args_parsed.log_level in constants.LOG_LEVELS
), "The specified `log-level` is not an allowed value."
assert isinstance(
args_parsed.page_size, int
), "Argument `page-size` must be an int."
arguments = MainArguments(
client_key=args_parsed.client_key,
client_secret=args_parsed.client_secret,
output_directory=args_parsed.output_directory,
log_level=args_parsed.log_level,
page_size=args_parsed.page_size,
input_directory=args_parsed.input_directory,
sync_database_directory=args_parsed.sync_database_directory,
extract_activities=constants.Features.Activities in args_parsed.feature,
extract_assignments=constants.Features.Assignments in args_parsed.feature,
extract_attendance=constants.Features.Attendance in args_parsed.feature,
extract_grades=constants.Features.Grades in args_parsed.feature,
)
return arguments
