def main() -> None:
"""
Command-line entry point.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
gate_home = get_envvar_or_die(EnvVar.GATE_HOME)
plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE)
kcl_pharmacotherapy_dir = get_envvar_or_die(EnvVar.KCL_PHARMACOTHERAPY_DIR)
check_call_verbose([
"java",
"-classpath",
f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*",
f"-Dgate.home={gate_home}",
"CrateGatePipeline",
"--pluginfile",
plugin_file,
"--gate_app",
os.path.join(kcl_pharmacotherapy_dir, "application.xgapp"),
"--include_set",
"Output",
"--annotation",
"Prescription",
"--input_terminator",
DEMO_NLP_INPUT_TERMINATOR,
"--output_terminator",
DEMO_NLP_OUTPUT_TERMINATOR,
"--suppress_gate_stdout",
"--show_contents_on_crash",
"-v",
"-v",
])
def main() -> None:
"""
Command line entry point.
"""
description = ("Print demo config file or demo processor constants file "
"for server side cloud nlp.")
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.RawDescriptionHelpFormatter)
arg_group = parser.add_mutually_exclusive_group()
arg_group.add_argument(
"--config",
action="store_true",
help="Print a demo config file for server side cloud nlp.")
arg_group.add_argument(
"--processors",
action="store_true",
help="Print a demo processor constants file for server side cloud "
"nlp.")
args = parser.parse_args()
main_only_quicksetup_rootlogger()
if args.config:
print(demo_config())
elif args.processors:
print(demo_processors())
else:
log.error("One option required: '--config' or '--processors'.")
def main() -> None:
"""
Command-line entry point.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
gate_home = get_envvar_or_die(EnvVar.GATE_HOME)
plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE)
kconnect_dir = get_envvar_or_die(EnvVar.KCL_KCONNECT_DIR)
check_call_verbose([
"java",
"-classpath",
f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*",
f"-Dgate.home={gate_home}",
"CrateGatePipeline",
"--pluginfile",
plugin_file,
"--gate_app",
os.path.join(kconnect_dir, "main-bio", "main-bio.xgapp"),
"--annotation",
"Disease_or_Syndrome",
"--input_terminator",
DEMO_NLP_INPUT_TERMINATOR,
"--output_terminator",
DEMO_NLP_OUTPUT_TERMINATOR,
"--suppress_gate_stdout",
"--show_contents_on_crash",
"-v",
"-v",
])
def main() -> None:
"""
Command-line entry point.
"""
parser = argparse.ArgumentParser(
description="Tool to initialize the database used by CRATE's "
"implementation of an NLPRP server.")
parser.add_argument(
"config_uri",
type=str,
help="Config file to read (e.g. 'development.ini'); URL of database "
"is found here.")
args = parser.parse_args()
main_only_quicksetup_rootlogger()
config_file = args.config_uri
log.debug(f"Settings file: {config_file}")
settings = get_appsettings(config_file)
engine = engine_from_config(settings,
NlpServerConfigKeys._SQLALCHEMY_PREFIX)
sqla_url = get_safe_url_from_engine(engine)
log.info(f"Using database {sqla_url!r}")
dbsession.configure(bind=engine)
log.info("Creating database structure...")
Base.metadata.create_all(engine)
log.info("... done.")
def main() -> None:
"""
Command-line entry point. See command-line help.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
parser = argparse.ArgumentParser(description="Reformat source files")
parser.add_argument("--rewrite",
action="store_true",
help="Rewrite the files")
parser.add_argument("--show",
action="store_true",
help="Show the files to stdout")
parser.add_argument(
"--process_only_filenum",
type=int,
help="Only process this file number (1-based index) in each "
"directory; for debugging only")
args = parser.parse_args()
rewrite = args.rewrite
show_only = args.show
process_only_filenum = args.process_only_filenum
if not rewrite and not show_only:
log.info("Not rewriting and not showing; will just catalogue files "
"and report things it's changing")
reformat_python_docstrings(top_dirs=[CRATE_ROOT_DIR],
show_only=show_only,
rewrite=rewrite,
process_only_filenum=process_only_filenum,
correct_copyright_lines=CORRECT_COPYRIGHT_LINES)
def main() -> None:
"""
Command-line entry point.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
gate_home = get_envvar_or_die(EnvVar.GATE_HOME)
plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE)
kcl_lewy_dir = get_envvar_or_die(EnvVar.KCL_LEWY_BODY_DIAGNOSIS_DIR)
check_call_verbose([
"java",
"-classpath",
f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*",
f"-Dgate.home={gate_home}",
"CrateGatePipeline",
"--pluginfile",
plugin_file,
"--gate_app",
os.path.join(kcl_lewy_dir, "application.xgapp"),
"--set_annotation",
"",
"DiagnosisAlmost",
"--set_annotation",
"Automatic",
"cDiagnosis",
"--input_terminator",
DEMO_NLP_INPUT_TERMINATOR,
"--output_terminator",
DEMO_NLP_OUTPUT_TERMINATOR,
"--suppress_gate_stdout",
"--show_contents_on_crash",
"-v",
"-v",
])
def main():
"""
Command-line processor. See ``--help`` for details.
"""
main_only_quicksetup_rootlogger()
parser = argparse.ArgumentParser()
parser.add_argument(
"filenames",
nargs="+",
help="Names of CSV/TSV files to merge"
)
parser.add_argument(
"--outfile",
default="-",
help="Specify an output filename. If omitted or '-', stdout is used.",
)
parser.add_argument(
"--inputdialect",
default="excel",
help="The input files' CSV/TSV dialect. Default: %(default)s.",
choices=csv.list_dialects(),
)
parser.add_argument(
"--outputdialect",
default="excel",
help="The output file's CSV/TSV dialect. Default: %(default)s.",
choices=csv.list_dialects(),
)
parser.add_argument(
"--noheaders",
action="store_true",
help="By default, files are assumed to have column headers. "
"Specify this option to assume no headers.",
)
parser.add_argument(
"--debug",
action="store_true",
help="Verbose debugging output.",
)
progargs = parser.parse_args()
kwargs = {
"filenames": progargs.filenames,
"input_dialect": progargs.inputdialect,
"output_dialect": progargs.outputdialect,
"debug": progargs.debug,
"headers": not progargs.noheaders,
}
if progargs.outfile == '-':
log.info("Writing to stdout")
merge_csv(outfile=sys.stdout, **kwargs)
else:
log.info("Writing to " + repr(progargs.outfile))
with open(progargs.outfile, 'w') as outfile:
# noinspection PyTypeChecker
merge_csv(outfile=outfile, **kwargs)
def meta_main() -> None:
"""
Command-line process for ``camcops_server_meta`` tool.
"""
parser = argparse.ArgumentParser(
description="Run commands across multiple CamCOPS databases")
parser.add_argument('cc_command',
type=str,
help="Main command to pass to CamCOPS")
parser.add_argument('--filespecs',
nargs='+',
required=True,
help="List of CamCOPS config files (wildcards OK)")
parser.add_argument(
'--ccargs',
nargs='*',
help="List of CamCOPS arguments, to which '--' will be prefixed")
parser.add_argument('--python',
default=sys.executable,
help="Python interpreter (default: {})".format(
sys.executable))
parser.add_argument(
'--camcops',
default=DEFAULT_CAMCOPS,
help="CamCOPS server executable (default: {})".format(DEFAULT_CAMCOPS))
parser.add_argument('-d',
'--dummyrun',
action="store_true",
help="Dummy run (show filenames only)")
parser.add_argument('-v', '--verbose', action="store_true", help="Verbose")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
log.debug("Arguments: {}", args)
# Delayed import so --help doesn't take ages
from camcops_server.camcops_server import main as camcops_main # delayed import # noqa
did_something = False
# old_sys_argv = sys.argv.copy()
for filespec in args.filespecs:
for filename in glob.glob(filespec):
did_something = True
log.info("Processing: {}", filename)
sys.argv = ([
'camcops_server', # dummy argv[0]
args.cc_command,
"--config",
filename
] + ['--{}'.format(x) for x in args.ccargs or []])
log.debug("Executing command: {}", sys.argv)
if args.dummyrun:
continue
camcops_main() # using the new sys.argv
# subprocess.check_call(cmdargs)
if not did_something:
log.info("Nothing to do; no files found")
def main() -> NoReturn:
"""
Command-line processor. See ``--help`` for details.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"input_file", help="Input PDF (which is not modified by this program)")
parser.add_argument("output_file", help="Output PDF")
parser.add_argument(
"--slice_horiz",
type=int,
default=1,
help="Slice the input PDF first into this many parts horizontally")
parser.add_argument(
"--slice_vert",
type=int,
default=1,
help="Slice the input PDF first into this many parts vertically")
parser.add_argument(
"--longedge",
action="store_true",
help="Create PDF for long-edge duplex printing, not short edge")
parser.add_argument("--overwrite",
action="store_true",
help="Allow overwriting of an existing output file")
parser.add_argument(
"--unittest",
action="store_true",
help="Run unit tests and exit (you must pass dummy values for "
"input/output files to use these tests)")
# ... because requiring dummy input/output filenames for unit testing
# is less confusing for the majority of users than showing syntax in
# which they are optional!
args = parser.parse_args()
if args.unittest:
log.warning("Performing unit tests")
# unittest.main() doesn't play nicely with argparse; they both
# use sys.argv by default (and we end up with what looks like garbage
# from the argparse help facility); but this works:
unittest.main(argv=[sys.argv[0]])
sys.exit(EXIT_SUCCESS)
success = convert_to_foldable(
input_filename=os.path.abspath(args.input_file),
output_filename=os.path.abspath(args.output_file),
slice_horiz=args.slice_horiz,
slice_vert=args.slice_vert,
overwrite=args.overwrite,
longedge=args.longedge)
sys.exit(EXIT_SUCCESS if success else EXIT_FAILURE)
def main() -> None:
"""
Command-line entry point.
"""
description = "Manage users for the CRATE nlp_web server."
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.RawDescriptionHelpFormatter)
arg_group = parser.add_mutually_exclusive_group()
arg_group.add_argument("--adduser",
nargs=2,
metavar=("USERNAME", "PASSWORD"),
help="Add a user and associated password.")
arg_group.add_argument("--rmuser",
nargs=1,
metavar="USERNAME",
help="Remove a user by specifying their username.")
arg_group.add_argument("--changepw",
nargs=2,
metavar=("USERNAME", "PASSWORD"),
help="Change a user's password.")
args = parser.parse_args()
main_only_quicksetup_rootlogger()
if not args.adduser and not args.rmuser and not args.changepw:
log.error(
"One option required: '--adduser', '--rmuser' or '--changepw'.")
return
log.debug(f"Settings file: {SETTINGS_PATH}")
log.debug(f"Users file: {USERS_FILENAME}")
if args.rmuser:
username = args.rmuser[0]
proceed = input(f"Confirm remove user: {username} ? [yes/no] ")
if proceed.lower() == "yes":
rm_user(username)
else:
log.info("User remove aborted.")
elif args.adduser:
username = args.adduser[0]
password = args.adduser[1]
add_user(username, password)
elif args.changepw:
username = args.changepw[0]
new_password = args.changepw[1]
proceed = input(
f"Confirm change password for user: {username} ? [yes/no] ")
if proceed.lower() == "yes":
change_password(username, new_password)
else:
log.info("Password change aborted.")
def main() -> None:
"""
Command-line processor. See ``--help`` for details.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument("directory", nargs="?", default=os.getcwd())
parser.add_argument("--reportevery", default=10000)
args = parser.parse_args()
log.info("Extensions in directory {!r}:", args.directory)
print("\n".join(
repr(x) for x in list_file_extensions(args.directory,
reportevery=args.reportevery)))
def main() -> None:
main_only_quicksetup_rootlogger()
process("test1_equal_preferences_check_output_consistency.xlsx",
"test_out1.xlsx")
process("test2_trivial_perfect.xlsx", "test_out2.xlsx")
process("test3_n60_two_equal_solutions.xlsx", "test_out3.xlsx")
process("test4_n10_multiple_ties.xlsx", "test_out4.xlsx")
process("test5_mean_vs_variance.xlsx", "test_out5.xlsx")
process("test6_multiple_students_per_project.xlsx", "test_out6.xlsx")
process("test7_eligibility.xlsx", "test_out7.xlsx")
process("test8_tied_preferences.xlsx", "test_out8.xlsx",
["--allow_supervisor_preference_ties"])
process("test9_supervisor_constraints.xlsx", "test_out9.xlsx",
["--debug_model", "--no_shuffle"])
def main() -> None:
"""
Command-line entry point.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
gate_home = get_envvar_or_die(EnvVar.GATE_HOME)
log.info(f"Note the unrealistic use of {DEMO_NLP_INPUT_TERMINATOR!r} as "
f"an end-of-input marker. Don't use that for real!")
check_call_verbose([
"java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*",
f"-Dgate.home={gate_home}", "CrateGatePipeline", "--annotation",
"Person", "--annotation", "Location", "--input_terminator",
DEMO_NLP_INPUT_TERMINATOR, "--output_terminator",
DEMO_NLP_OUTPUT_TERMINATOR, "--log_tag", ".", "-v", "-v", "--demo"
])
def main() -> None:
"""
Command-line entry point.
"""
main_only_quicksetup_rootlogger(level=logging.DEBUG)
medex_home = get_envvar_or_die(EnvVar.MEDEX_HOME)
check_call_verbose([
"java",
"-classpath",
f"{CrateDir.JAVA_CLASSES}:{medex_home}/bin:{medex_home}/lib/*",
"CrateMedexPipeline",
"--help",
"-v",
"-v",
])
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--make",
action="store_true",
help="Do things! Otherwise will just show its intent.")
parser.add_argument("--destroy_first",
action="store_true",
help="Destroy all existing autodocs first")
parser.add_argument("--verbose", action="store_true", help="Be verbose")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
make_autodoc(make=args.make, destroy_first=args.destroy_first)
def main() -> None:
"""
Command-line entry point.
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"source_codes", nargs="+", type=int,
help="Source codes to look up "
"(along with their descendants if --descendants is specified)"
)
parser.add_argument(
"--descendants", action="store_true",
help="Include descendants of the codes specified"
)
parser.add_argument(
"--concept", type=str, default=DEFAULT_CONCEPT,
help="Athena OHDSI CONCEPT.csv TSV file including the source and "
"destination vocabularies"
)
parser.add_argument(
"--concept_relationship", type=str,
default=DEFAULT_CONCEPT_RELATIONSHIP,
help="Athena OHDSI CONCEPT_RELATIONSHIP.csv TSV file "
"including the source and destination vocabularies"
)
parser.add_argument(
"--src_vocabulary", type=str, default=AthenaVocabularyId.SNOMED,
help="Source vocabulary"
)
parser.add_argument(
"--dest_vocabulary", type=str, default=AthenaVocabularyId.OPCS4,
help="Destination vocabulary"
)
args = parser.parse_args()
main_only_quicksetup_rootlogger()
print_equivalent_opcs_codes(
source_codes=args.source_codes,
source_vocabulary=args.src_vocabulary,
destination_vocabulary=args.dest_vocabulary,
concept_file=args.concept,
concept_relationship_file=args.concept_relationship,
with_descendants=args.descendants,
)
def main() -> None:
main_only_quicksetup_rootlogger(level=logging.DEBUG, with_thread_id=True)
if PROFILE:
pr = cProfile.Profile()
pr.enable()
test_two_antidepressant_episodes()
pr.disable()
s = io.StringIO()
sortby = 'cumulative'
# In Python 3.7: sortby = pstats.SortKey.CUMULATIVE
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats(50) # top 50
print(s.getvalue())
else:
test_two_antidepressant_episodes()
def main() -> None:
main_only_quicksetup_rootlogger(level=logging.DEBUG)
# Define jobs
possible_jobs = ["experiment_1", "experiment_1b", "experiment_2"]
# Command-line arguments
parser = argparse.ArgumentParser(
description="Launch spread-modelling job on SLURM cluster",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("jobs",
type=str,
nargs="+",
choices=possible_jobs + ["all"],
help="Job code")
args = parser.parse_args()
# Launch
jobs = possible_jobs if "all" in args.jobs else args.jobs
for job in jobs:
launch(job)
def main() -> None:
parser = argparse.ArgumentParser(
description="Parse a mailbox file from stdin to stdout, "
"removing any attachments",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("input", type=str, help="Filename for input")
parser.add_argument("output", type=str, help="Filename for output")
parser.add_argument("--verbose",
"-v",
action="store_true",
help="Be verbose")
parser.add_argument("--report",
type=int,
default=100,
help="Report every n messages")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
if os.path.exists(args.output):
errmsg = f"Output file exists: {args.output}"
log.critical(errmsg)
raise ValueError(errmsg)
log.info("Opening input file: {filename} ({size})",
filename=args.input,
size=bytes2human(os.path.getsize(args.input)))
input_box = mailbox.mbox(args.input, create=False)
log.info("Opening output file: {} (new file)", args.output)
output_box = mailbox.mbox(args.output, create=True)
log.info("Processing messages...")
msg_count = 0
for message in input_box.itervalues():
msg_count += 1
if msg_count % args.report == 0:
log.debug("Processing message {}", msg_count)
processed_msg = clean_message(message, topmost=True)
output_box.add(processed_msg)
log.info("Done; processed {} messages.", msg_count)
log.info("Output size: {}", bytes2human(os.path.getsize(args.output)))
def make_wsgi_app(global_config: Dict[Any, Any], **settings) -> Router:
# Logging
main_only_quicksetup_rootlogger()
logging.getLogger('sqlalchemy').setLevel(logging.WARNING)
# log.debug(f"global_config: {global_config!r}")
# log.debug(f"settings: {settings!r}")
# Database
engine = engine_from_config(settings,
NlpServerConfigKeys._SQLALCHEMY_PREFIX,
**SQLALCHEMY_COMMON_OPTIONS)
# ... add to config - pool_recycle is set to create new sessions every 7h
sqla_url = get_safe_url_from_engine(engine)
log.info(f"Using database {sqla_url!r}")
dbsession.configure(bind=engine)
Base.metadata.bind = engine
# Pyramid
config = Configurator(settings=settings)
# Security policies
authn_policy = AuthTktAuthenticationPolicy(
settings[NlpServerConfigKeys.NLP_WEBSERVER_SECRET],
secure=True, # only allow requests over HTTPS
hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
# Compression
config.add_tween("cardinal_pythonlib.pyramid.compression.CompressionTweenFactory") # noqa
# Routes
config.add_route('index', '/')
config.scan('.views')
# Create WSGI app
return config.make_wsgi_app()
def main() -> None:
"""
Command-line processor. See ``--help`` for details.
"""
parser = ArgumentParser(description="Remove duplicate files")
parser.add_argument(
"directory",
nargs="+",
help="Files and/or directories to check and remove duplicates from.")
parser.add_argument("--recursive",
action="store_true",
help="Recurse through any directories found")
parser.add_argument("--dummy_run",
action="store_true",
help="Dummy run only; don't actually delete anything")
parser.add_argument(
"--run_repeatedly",
type=int,
help="Run the tool repeatedly with a pause of <run_repeatedly> "
"seconds between runs. (For this to work well,"
"you should specify one or more DIRECTORIES in "
"the 'filename' arguments, not files, and you will need the "
"--recursive option.)")
parser.add_argument("--verbose",
action="store_true",
help="Verbose output")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
while True:
deduplicate(args.directory,
recursive=args.recursive,
dummy_run=args.dummy_run)
if args.run_repeatedly is None:
break
log.info("Sleeping for {} s...", args.run_repeatedly)
sleep(args.run_repeatedly)
"""
import argparse
import logging
from cardinal_pythonlib.logs import main_only_quicksetup_rootlogger
from crate_anon.nlp_manager import (
all_processors,
regex_parser,
regex_units,
)
def test_all_regex_nlp(verbose: bool = False) -> None:
"""
Test all NLP-related regular expressions.
"""
regex_parser.test_all(verbose=verbose) # basic regexes
regex_units.test_all(verbose=verbose)
all_processors.test_all_processors(verbose=verbose)
# ... tests all parser classes
if __name__ == '__main__':
main_only_quicksetup_rootlogger(level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', '-v', action="store_true", help="Verbose")
args = parser.parse_args()
test_all_regex_nlp(verbose=args.verbose)
def main() -> None:
"""
Creates an Alembic database migration for CamCOPS, by comparing the
metadata (from Python) with the current database.
Note special difficulty with "variant" types, such as
``Integer().with_variant(...)`` which are (2017-08-21, alembic==0.9.4)
rendered as ``sa.Variant()`` only with a MySQL backend.
- https://bitbucket.org/zzzeek/alembic/issues/433/variant-base-not-taken-into-account-when
- https://bitbucket.org/zzzeek/alembic/issues/131/create-special-rendering-for-variant
We deal with these via
:func:`camcops_server.alembic.env.process_revision_directives` in
``env.py``.
""" # noqa
desc = (
"Create database revision. Note:\n"
"- The config used will be that from the environment variable {} "
"(currently {!r}).\n"
"- Alembic compares (a) the current state of the DATABASE to (b) "
"the state of the SQLAlchemy metadata (i.e. the CODE). It creates a "
"migration file to change the database to match the code.\n"
"- Accordingly, in the rare event of wanting to do a fresh start, you "
"need an *empty* database.\n"
"- More commonly, you want a database that is synced to a specific "
"Alembic version (with the correct structure, and the correct version "
"in the alembic_version table). If you have made manual changes, such "
"that the actual database structure doesn't match the structure that "
"Alembic expects based on that version, there's likely to be "
"trouble.\n".format(
ENVVAR_CONFIG_FILE, environ.get(ENVVAR_CONFIG_FILE, None)
)
)
wrapped = "\n\n".join(
textwrap.fill(x, width=79, initial_indent="", subsequent_indent=" ")
for x in desc.splitlines()
)
# noinspection PyTypeChecker
parser = ArgumentParser(
description=wrapped, formatter_class=RawDescriptionHelpFormatter
)
parser.add_argument("message", help="Revision message")
parser.add_argument("--verbose", action="store_true", help="Be verbose")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO
)
# ... hmpf; ignored (always debug); possible Alembic forces this.
# Check the existing database version is OK.
config = get_default_config_from_os_env()
config.assert_database_ok()
# Then, if OK, create an upgrade.
create_database_migration_numbered_style(
alembic_ini_file=ALEMBIC_INI_FILE,
alembic_versions_dir=ALEMBIC_VERSIONS_DIR,
message=args.message,
n_sequence_chars=N_SEQUENCE_CHARS,
)
print(
r"""
Now:
- Check the new migration file.
- Check in particular for incorrectly dialect-specific stuff, e.g. with
grep "mysql\." *.py
... should only show "sa.SOMETYPE().with_variant(mysql.MYSQLTYPE..."
- and
grep "sa\.Variant" *.py
... suggests an error that should be "Sometype().with_variant(...)"; see
source here.
"""
)
def main():
"""
Command-line entry point.
"""
check_prerequisites()
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(
description="""
- Creates a Debian (.deb) and RPM (.rpm) distribution file for the CamCOPS
server, for distribution under Linux.
- In brief, the following sequence is followed as the package is built:
* The CamCOPS server is packaged up from source using
python setup.py sdist
and zipped in a Debian-safe way.
* The principle is that the Python package should do all the work, not the
Debian framework. This also means that a user who elects to install via pip
gets exactly the same functional file structure.
* A Debian package is built, containing all the usual Debian goodies (man
packages, the preinst/postinst/prerm/postrm files, cataloguing and control
files, etc.).
* The package is checked with Lintian.
* An RPM is built from the .deb package.
- The user then installs the DEB or RPM file. In addition to installing
standard things like man pages, this then:
* attempts to stop supervisord for the duration of the installation (because
that's the usual way to run a CamCOPS server);
* creates a few standard directories (e.g. for CamCOPS configuration and
lock files), including
{LINUX_DEFAULT_CAMCOPS_CONFIG_DIR}
{LINUX_DEFAULT_CAMCOPS_DIR}
{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}
{LINUX_DEFAULT_LOCK_DIR}
* checks that Python is available on the system;
* uses the system Python to create a Python virtual environment within
{LINUX_DEFAULT_CAMCOPS_DIR};
* uses the virtual environment's "pip" command to install the distributed
CamCOPS Python package within that virtual environment;
... which also appears to compile .py to .pyc files automatically;
* creates master executable scripts (which call corresponding Python
scripts):
{DSTCONSOLEFILE}
{DSTMETACONSOLEFILE}
* sets some permissions (to default users such as "www-data" on Ubuntu, or
"apache" on CentOS);
* restarts supervisord.
""".format(
DSTCONSOLEFILE=DSTCONSOLEFILE,
DSTMETACONSOLEFILE=DSTMETACONSOLEFILE,
LINUX_DEFAULT_CAMCOPS_CONFIG_DIR=LINUX_DEFAULT_CAMCOPS_CONFIG_DIR,
LINUX_DEFAULT_CAMCOPS_DIR=LINUX_DEFAULT_CAMCOPS_DIR,
LINUX_DEFAULT_LOCK_DIR=LINUX_DEFAULT_LOCK_DIR,
LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR=
LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR, # noqa
),
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--verbose", "-v", action="store_true")
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
log.info("mainversion: {}", MAINVERSION)
log.info("changedate: {}", CHANGEDATE)
build_package()
run_cmd(["crate_django_manage", "help", "lookup_patient"],
join(WEB_DIR, "_crate_django_manage_lookup_patient_help.txt"))
run_cmd(["crate_django_manage", "help", "populate"],
join(WEB_DIR, "_crate_django_manage_populate_help.txt"))
run_cmd([
"crate_django_manage", "help", "resubmit_unprocessed_tasks"
], join(
WEB_DIR,
"_crate_django_manage_resubmit_unprocessed_tasks_help.txt")) # noqa
run_cmd(["crate_django_manage", "help", "runcpserver"],
join(WEB_DIR, "_crate_django_manage_runcpserver_help.txt"))
run_cmd(["crate_django_manage", "help", "runserver"],
join(WEB_DIR, "_crate_django_manage_runserver_help.txt"))
run_cmd(["crate_django_manage", "help", "test_email"],
join(WEB_DIR, "_crate_django_manage_test_email_help.txt"))
run_cmd(["crate_launch_celery", "--help"],
join(WEB_DIR, "_crate_launch_celery_help.txt"))
run_cmd(["crate_launch_django_server", "--help"],
join(WEB_DIR, "_crate_launch_django_server_help.txt"))
run_cmd(["crate_print_demo_crateweb_config"],
join(WEB_DIR, "_specimen_web_config.py"))
log.warning("Skipping crate_windows_service_help.txt (requires Windows)")
log.info("Done.")
if __name__ == "__main__":
main_only_quicksetup_rootlogger()
main()
def main() -> None:
"""
Command-line processor. See ``--help`` for details.
"""
main_only_quicksetup_rootlogger()
wdcd_suffix = "_with_drop_create_database"
timeformat = "%Y%m%dT%H%M%S"
parser = argparse.ArgumentParser(
description=f"""
Back up a specific MySQL database. The resulting filename has the
format '<DATABASENAME>_<DATETIME><SUFFIX>.sql', where <DATETIME> is of
the ISO-8601 format {timeformat!r} and <SUFFIX> is either blank or
{wdcd_suffix!r}.
A typical filename is therefore 'mydb_20190415T205524.sql'.
""",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("databases", nargs="+", help="Database(s) to back up")
parser.add_argument("--max_allowed_packet",
default="1GB",
help="Maximum size of buffer")
parser.add_argument("--mysqldump",
default="mysqldump",
help="mysqldump executable")
parser.add_argument("--username", default="root", help="MySQL user")
parser.add_argument(
"--password",
help="MySQL password (AVOID THIS OPTION IF POSSIBLE; VERY INSECURE; "
"VISIBLE TO OTHER PROCESSES; if you don't use it, you'll be "
"prompted for the password)")
parser.add_argument(
"--with_drop_create_database",
action="store_true",
help="Include DROP DATABASE and CREATE DATABASE commands, and append "
"a suffix to the output filename as above")
parser.add_argument(
"--output_dir",
type=str,
help="Output directory (if not specified, current directory will be "
"used)")
parser.add_argument("--verbose",
action="store_true",
help="Verbose output")
args = parser.parse_args()
output_dir = args.output_dir or os.getcwd()
os.chdir(output_dir)
password = args.password or getpass.getpass(
prompt=f"MySQL password for user {args.username}: ")
output_files = [] # type: List[str]
if args.with_drop_create_database:
log.info("Note that the DROP DATABASE commands will look like they're "
"commented out, but they're not: "
"https://dba.stackexchange.com/questions/59294/")
suffix = wdcd_suffix
else:
suffix = ""
for db in args.databases:
now = datetime.datetime.now().strftime(timeformat)
outfilename = f"{db}_{now}{suffix}.sql"
display_args = cmdargs(
mysqldump=args.mysqldump,
username=args.username,
password=password,
database=db,
verbose=args.verbose,
with_drop_create_database=args.with_drop_create_database,
max_allowed_packet=args.max_allowed_packet,
hide_password=True)
actual_args = cmdargs(
mysqldump=args.mysqldump,
username=args.username,
password=password,
database=db,
verbose=args.verbose,
with_drop_create_database=args.with_drop_create_database,
max_allowed_packet=args.max_allowed_packet,
hide_password=False)
log.info("Executing: " + repr(display_args))
log.info("Output file: " + repr(outfilename))
try:
with open(outfilename, "w") as f:
subprocess.check_call(actual_args, stdout=f)
except subprocess.CalledProcessError:
os.remove(outfilename)
log.critical("Failed!")
sys.exit(1)
output_files.append(outfilename)
log.info("Done. See:\n" + "\n".join(" " + x for x in output_files))
if args.with_drop_create_database:
log.info("To restore: mysql -u USER -p < BACKUP.sql")
else:
log.info("To restore: mysql -u USER -p DATABASE < BACKUP.sql")
def main() -> None:
"""
Command-line entry point.
"""
# Parser
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter, )
parser.add_argument("--cachepath",
type=str,
default=DEFAULT_CACHE_PATH,
help="Cache path for ChEBI files")
parser.add_argument("--verbose", action="store_true", help="Be verbose")
subparsers = parser.add_subparsers(title="subcommands",
description="Valid subcommands",
dest="command")
subparsers.required = True
def add_exact(p: argparse.ArgumentParser) -> None:
p.add_argument("--exact_search",
dest="exact_search",
action="store_true",
help="Search using exact term")
p.add_argument("--inexact_search",
dest="exact_search",
action="store_false",
help="Search allowing inexact matches")
p.set_defaults(exact_search=DEFAULT_EXACT_SEARCH)
p.add_argument("--exact_match",
dest="exact_match",
action="store_true",
help="Return results for exact term only")
p.add_argument("--inexact_match",
dest="exact_match",
action="store_false",
help="Return results allowing inexact matches")
p.set_defaults(exact_match=DEFAULT_EXACT_MATCH)
def add_entities(p: argparse.ArgumentParser) -> None:
p.add_argument("entity",
type=str,
nargs="+",
help="Entity or entities to search for")
# -------------------------------------------------------------------------
# Test
# -------------------------------------------------------------------------
parser_test = subparsers.add_parser("test", help="Run some simple tests")
parser_test.set_defaults(func=lambda args: testfunc1())
# -------------------------------------------------------------------------
# Search
# -------------------------------------------------------------------------
parser_search = subparsers.add_parser(
"search", help="Search for an entity in the ChEBI database")
add_entities(parser_search)
add_exact(parser_search)
parser_search.set_defaults(func=lambda args: search_and_list_multiple(
search_terms=args.entity,
exact_search=args.exact_search,
exact_match=args.exact_match,
))
# -------------------------------------------------------------------------
# Describe
# -------------------------------------------------------------------------
parser_describe = subparsers.add_parser(
"describe", help="Describe an entity/entities in the ChEBI database")
add_entities(parser_describe)
add_exact(parser_describe)
parser_describe.set_defaults(
func=lambda args: search_and_describe_multiple( # noqa
search_terms=args.entity,
exact_search=args.exact_search,
exact_match=args.exact_match,
))
# -------------------------------------------------------------------------
# Ancestors
# -------------------------------------------------------------------------
parser_ancestors = subparsers.add_parser(
"ancestors",
help="Show ancestors of an entity/entities in the ChEBI database")
add_entities(parser_ancestors)
parser_ancestors.add_argument(
"--relationships",
type=str,
nargs="+",
default=DEFAULT_ANCESTOR_RELATIONSHIPS,
help="Relationship types that define an ancestor")
parser_ancestors.add_argument(
"--max_generations",
type=int,
default=None,
help="Number of generations to search, or None for unlimited")
parser_ancestors.set_defaults(func=lambda args: report_ancestors_multiple(
entity_names=args.entity,
relationships=args.relationships,
max_generations=args.max_generations,
))
# -------------------------------------------------------------------------
# Categorize
# -------------------------------------------------------------------------
parser_categorize = subparsers.add_parser(
"categorize", help="Categorize a list of drugs.")
parser_categorize.add_argument(
"--entities",
type=str,
required=True,
help="Input file, one entity (e.g. drug) name per line.")
parser_categorize.add_argument(
"--categories",
type=str,
required=True,
help="Name of file containing categories, one per line "
"(earlier categories preferred to later).")
parser_categorize.add_argument(
"--entity_synonyms",
type=str,
default=None,
help="Name of CSV file (with optional # comments) containing synonyms "
"in the format 'entity_from, entity_to'")
parser_categorize.add_argument(
"--category_synonyms",
type=str,
default=None,
help="Name of CSV file (with optional # comments) containing synonyms "
"in the format 'category_from, category_to'. The translation is "
"applied to ChEBI categories before matching. For example you "
"can map 'EC 3.1.1.7 (acetylcholinesterase) inhibitor' to "
"'acetylcholinesterase inhibitor' and then use only "
"'acetylcholinesterase inhibitor' in your category file.")
parser_categorize.add_argument(
"--manual_categories",
type=str,
default=None,
help="Name of CSV file (with optional # comments) containing manual "
"categorizations in the format 'entity, category'")
parser_categorize.add_argument("--results",
type=str,
required=True,
help="Output CSV file.")
parser_categorize.add_argument(
"--relationships",
type=str,
nargs="+",
default=DEFAULT_ANCESTOR_RELATIONSHIPS,
help="Relationship types that define an ancestor")
parser_categorize.set_defaults(func=lambda args: categorize_from_file(
entity_filename=args.entities,
results_filename=args.results,
category_filename=args.categories,
entity_synonyms_filename=args.entity_synonyms,
category_synonyms_filename=args.category_synonyms,
manual_categories_filename=args.manual_categories,
relationships=args.relationships,
))
# -------------------------------------------------------------------------
# Parse and run
# -------------------------------------------------------------------------
cmdargs = parser.parse_args()
# Logging
main_only_quicksetup_rootlogger(
level=logging.DEBUG if cmdargs.verbose else logging.INFO)
log.debug(f"ChEBI lookup from cardinal_pythonlib=={VERSION_STRING}")
# Caching
log.debug(f"Using cache path: {cmdargs.cachepath}")
set_download_cache_path(cmdargs.cachepath)
# Do something useful
cmdargs.func(cmdargs)
def main() -> None:
"""
Command-line handler for the ``pause_process_by_disk_space`` tool.
Use the ``--help`` option for help.
"""
parser = ArgumentParser(
description="Pauses and resumes a process by disk space; LINUX ONLY."
)
parser.add_argument(
"process_id", type=int,
help="Process ID."
)
parser.add_argument(
"--path", required=True,
help="Path to check free space for (e.g. '/')"
)
parser.add_argument(
"--pause_when_free_below", type=str, required=True,
help="Pause process when free disk space below this value (in bytes "
"or as e.g. '50G')"
)
parser.add_argument(
"--resume_when_free_above", type=str, required=True,
help="Resume process when free disk space above this value (in bytes "
"or as e.g. '70G')"
)
parser.add_argument(
"--check_every", type=int, required=True,
help="Check every n seconds (where this is n)"
)
parser.add_argument(
"--verbose", action="store_true",
help="Verbose output"
)
args = parser.parse_args()
main_only_quicksetup_rootlogger(
level=logging.DEBUG if args.verbose else logging.INFO)
minimum = human2bytes(args.pause_when_free_below)
maximum = human2bytes(args.resume_when_free_above)
path = args.path
process_id = args.process_id
period = args.check_every
pause_args = ["kill", "-STOP", str(process_id)]
resume_args = ["kill", "-CONT", str(process_id)]
assert minimum < maximum, "Minimum must be less than maximum"
log.info(
f"Starting: controlling process {process_id}; "
f"checking disk space every {period} s; "
f"will pause when free space on {path} "
f"is less than {sizeof_fmt(minimum)} and "
f"resume when free space is at least {sizeof_fmt(maximum)}; "
f"pause command will be {pause_args}; "
f"resume command will be {resume_args}."
)
log.debug("Presuming that the process is RUNNING to begin with.")
paused = False
while True:
if not is_running(process_id):
log.info("Process {} is no longer running", process_id)
sys.exit(0)
space = shutil.disk_usage(path).free
log.debug("Disk space on {} is {}", path, sizeof_fmt(space))
if space < minimum and not paused:
log.info("Disk space down to {}: pausing process {}",
sizeof_fmt(space), process_id)
subprocess.check_call(pause_args)
paused = True
elif space >= maximum and paused:
log.info("Disk space up to {}: resuming process {}",
sizeof_fmt(space), process_id)
subprocess.check_call(resume_args)
paused = False
log.debug("Sleeping for {} seconds...", period)
sleep(period)
"plain/class member function of same name (unless "
"namespace is manually given)")
test(t.no_params_dogpile_default_fkg(), False)
t2 = SecondTestClass()
test(t.dogpile_default_test_2(), True)
test(t.dogpile_default_test_2(), False)
try:
test(t2.dogpile_default_test_2(), True)
log.info("dogpile.cache default FKG correctly distinguishing between "
"member functions of two different classes with same name")
except AssertionError:
log.warning("Known dogpile.cache default FKG problem - conflates "
"member functions of two different classes where "
"functions have same name (unless namespace is manually "
"given)")
test(t2.dogpile_default_test_2(), False)
log.info("Success!")
# TEST THIS WITH:
# python -m cardinal_pythonlib.dogpile_cache
if __name__ == '__main__':
level = logging.DEBUG if TESTING_VERBOSE else logging.INFO
if TESTING_USE_PRETTY_LOGS:
main_only_quicksetup_rootlogger(level=level)
else:
logging.basicConfig(level=level)
unit_test_cache()
def main() -> None:
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('mdbfile',
type=str,
help="Microsoft Access .MDB file to read")
parser.add_argument(
'--schemaonly',
action='store_true',
help="Print schema SQL only (otherwise: will create MySQL database and"
" write schema and data)")
parser.add_argument(
'--host',
type=str,
default='127.0.0.1', # not "localhost"
help="MySQL hostname or IP address")
parser.add_argument('--port',
type=int,
default=3306,
help="MySQL port number")
parser.add_argument('--user',
type=str,
default='root',
help="MySQL username")
parser.add_argument('--password',
type=str,
action=PasswordPromptAction,
help="MySQL password")
parser.add_argument('--mysqldb', type=str, help="MySQL database to use")
parser.add_argument('--create',
action="store_true",
help="Create database?")
args = parser.parse_args()
if not args.schemaonly and not args.mysqldb:
raise ValueError("Must specify --mysqldb, unless --schemaonly is used")
main_only_quicksetup_rootlogger(level=logging.DEBUG)
# -------------------------------------------------------------------------
log.info("Getting list of tables")
# -------------------------------------------------------------------------
tablecmd = ["mdb-tables", "-1", args.mdbfile]
# ... -1: one per line (or table names with spaces will cause confusion)
tables = get_command_output(tablecmd).splitlines()
tables.sort(key=lambda s: s.lower())
log.info("Tables: {}".format(tables))
# -------------------------------------------------------------------------
log.info("Fetching schema definition")
# -------------------------------------------------------------------------
schemacmd = [
"mdb-schema", # from Ubuntu package mdbtools
args.mdbfile, # database
"mysql", # backend
"--no-drop-table", # don't issue DROP TABLE statements (default)
"--not-null", # issue NOT NULL constraints (default)
"--no-default-values", # don't issue DEFAULT values (default)
"--indexes", # export INDEX statements (default)
"--relations", # export foreign key constraints (default)
]
schemasyntax = get_command_output(schemacmd)
# -------------------------------------------------------------------------
log.info("Converting any schema syntax oddities to MySQL syntax")
# -------------------------------------------------------------------------
# JAN 2013: Since my previous script, mdb-schema's mysql dialect has got
# much better. So, not much to do here.
# "COMMENT ON COLUMN" produced by mdb-schema and rejected by MySQL:
schemasyntax = re.sub("^COMMENT ON COLUMN.*$", "", schemasyntax, 0,
re.MULTILINE)
log.info("Schema syntax:")
print(schemasyntax)
# -------------------------------------------------------------------------
# Done?
# -------------------------------------------------------------------------
if args.schemaonly:
return
# -------------------------------------------------------------------------
log.info("Creating new database")
# -------------------------------------------------------------------------
createmysqldbcmd = [
"mysqladmin", "create", args.mysqldb, "--host={}".format(args.host),
"--port={}".format(args.port), "--user={}".format(args.user),
"--password={}".format(args.password)
]
# We could omit the actual password and the user would be prompted, but we
# need to send it this way later (see below), so this is not a huge
# additional security weakness!
# Linux/MySQL helpfully obscures the password in the "ps" list.
log.info(get_command_output(createmysqldbcmd))
# -------------------------------------------------------------------------
log.info("Sending schema command to MySQL")
# -------------------------------------------------------------------------
mysqlcmd = [
"mysql", "--host={}".format(args.host), "--port={}".format(args.port),
"--database={}".format(args.mysqldb), "--user={}".format(args.user),
"--password={}".format(args.password)
]
# Regrettably, we need the password here, as stdin will come from a pipe
print(get_pipe_series_output([mysqlcmd], schemasyntax))
# -------------------------------------------------------------------------
log.info("Copying data to MySQL")
# -------------------------------------------------------------------------
# For the data, we won't store the intermediate stuff in Python's memory,
# 'cos it's vast; I had one odd single-character mutation
# from "TimeInSession_ms" to "TimeInSession_mc" at row 326444 (perhaps
# therefore 37Mb or so into a long string).
# And I was trying to export ~1m records in that table alone.
# We'll use pipes instead and let the OS deal with the memory management.
# ... BUT (Jan 2013): now mdb-tools is better, text-processing not
# necessary - can use temporary disk file
# Turns out the bottleneck is the import to MySQL, not the export from MDB.
# So see http://dev.mysql.com/doc/refman/5.5/en/optimizing-innodb-bulk-data-loading.html # noqa
# The massive improvement is by disabling autocommit. (Example source
# database is 208M; largest table here is 554M as a textfile; it has
# 1,686,075 rows.) This improvement was from 20 Hz to the whole database
# in a couple of minutes (~13 kHz). Subsequent export from MySQL: takes a
# second or two to write whole DB (177M textfile).
for t in tables:
log.info("Processing table {}".format(t))
exportcmd = [
'mdb-export',
'-I',
'mysql', # -I backend: INSERT statements, not CSV
# MySQL's DATETIME field has this format: "YYYY-MM-DD HH:mm:SS"
# so we want this from the export:
'-D',
'%Y-%m-%d %H:%M:%S', # -D: date format
# ... don't put any extra quotes around it.
args.mdbfile, # database
t # table
]
with tempfile.NamedTemporaryFile(
mode="wt", encoding=sys.getdefaultencoding()) as outfile:
print("SET autocommit=0;", file=outfile)
subprocess.call(exportcmd, stdout=outfile)
print("\nCOMMIT;", file=outfile)
outfile.flush()
outfile.seek(0)
subprocess.call(mysqlcmd, stdin=outfile)
log.info("Finished.")
