def main() -> None: """ Command-line entry point. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) gate_home = get_envvar_or_die(EnvVar.GATE_HOME) plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE) kcl_pharmacotherapy_dir = get_envvar_or_die(EnvVar.KCL_PHARMACOTHERAPY_DIR) check_call_verbose([ "java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*", f"-Dgate.home={gate_home}", "CrateGatePipeline", "--pluginfile", plugin_file, "--gate_app", os.path.join(kcl_pharmacotherapy_dir, "application.xgapp"), "--include_set", "Output", "--annotation", "Prescription", "--input_terminator", DEMO_NLP_INPUT_TERMINATOR, "--output_terminator", DEMO_NLP_OUTPUT_TERMINATOR, "--suppress_gate_stdout", "--show_contents_on_crash", "-v", "-v", ])
def main() -> None: """ Command line entry point. """ description = ("Print demo config file or demo processor constants file " "for server side cloud nlp.") parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter) arg_group = parser.add_mutually_exclusive_group() arg_group.add_argument( "--config", action="store_true", help="Print a demo config file for server side cloud nlp.") arg_group.add_argument( "--processors", action="store_true", help="Print a demo processor constants file for server side cloud " "nlp.") args = parser.parse_args() main_only_quicksetup_rootlogger() if args.config: print(demo_config()) elif args.processors: print(demo_processors()) else: log.error("One option required: '--config' or '--processors'.")
def main() -> None: """ Command-line entry point. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) gate_home = get_envvar_or_die(EnvVar.GATE_HOME) plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE) kconnect_dir = get_envvar_or_die(EnvVar.KCL_KCONNECT_DIR) check_call_verbose([ "java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*", f"-Dgate.home={gate_home}", "CrateGatePipeline", "--pluginfile", plugin_file, "--gate_app", os.path.join(kconnect_dir, "main-bio", "main-bio.xgapp"), "--annotation", "Disease_or_Syndrome", "--input_terminator", DEMO_NLP_INPUT_TERMINATOR, "--output_terminator", DEMO_NLP_OUTPUT_TERMINATOR, "--suppress_gate_stdout", "--show_contents_on_crash", "-v", "-v", ])
def main() -> None: """ Command-line entry point. """ parser = argparse.ArgumentParser( description="Tool to initialize the database used by CRATE's " "implementation of an NLPRP server.") parser.add_argument( "config_uri", type=str, help="Config file to read (e.g. 'development.ini'); URL of database " "is found here.") args = parser.parse_args() main_only_quicksetup_rootlogger() config_file = args.config_uri log.debug(f"Settings file: {config_file}") settings = get_appsettings(config_file) engine = engine_from_config(settings, NlpServerConfigKeys._SQLALCHEMY_PREFIX) sqla_url = get_safe_url_from_engine(engine) log.info(f"Using database {sqla_url!r}") dbsession.configure(bind=engine) log.info("Creating database structure...") Base.metadata.create_all(engine) log.info("... done.")
def main() -> None: """ Command-line entry point. See command-line help. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) parser = argparse.ArgumentParser(description="Reformat source files") parser.add_argument("--rewrite", action="store_true", help="Rewrite the files") parser.add_argument("--show", action="store_true", help="Show the files to stdout") parser.add_argument( "--process_only_filenum", type=int, help="Only process this file number (1-based index) in each " "directory; for debugging only") args = parser.parse_args() rewrite = args.rewrite show_only = args.show process_only_filenum = args.process_only_filenum if not rewrite and not show_only: log.info("Not rewriting and not showing; will just catalogue files " "and report things it's changing") reformat_python_docstrings(top_dirs=[CRATE_ROOT_DIR], show_only=show_only, rewrite=rewrite, process_only_filenum=process_only_filenum, correct_copyright_lines=CORRECT_COPYRIGHT_LINES)
def main() -> None: """ Command-line entry point. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) gate_home = get_envvar_or_die(EnvVar.GATE_HOME) plugin_file = get_envvar_or_die(EnvVar.CRATE_GATE_PLUGIN_FILE) kcl_lewy_dir = get_envvar_or_die(EnvVar.KCL_LEWY_BODY_DIAGNOSIS_DIR) check_call_verbose([ "java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*", f"-Dgate.home={gate_home}", "CrateGatePipeline", "--pluginfile", plugin_file, "--gate_app", os.path.join(kcl_lewy_dir, "application.xgapp"), "--set_annotation", "", "DiagnosisAlmost", "--set_annotation", "Automatic", "cDiagnosis", "--input_terminator", DEMO_NLP_INPUT_TERMINATOR, "--output_terminator", DEMO_NLP_OUTPUT_TERMINATOR, "--suppress_gate_stdout", "--show_contents_on_crash", "-v", "-v", ])
def main(): """ Command-line processor. See ``--help`` for details. """ main_only_quicksetup_rootlogger() parser = argparse.ArgumentParser() parser.add_argument( "filenames", nargs="+", help="Names of CSV/TSV files to merge" ) parser.add_argument( "--outfile", default="-", help="Specify an output filename. If omitted or '-', stdout is used.", ) parser.add_argument( "--inputdialect", default="excel", help="The input files' CSV/TSV dialect. Default: %(default)s.", choices=csv.list_dialects(), ) parser.add_argument( "--outputdialect", default="excel", help="The output file's CSV/TSV dialect. Default: %(default)s.", choices=csv.list_dialects(), ) parser.add_argument( "--noheaders", action="store_true", help="By default, files are assumed to have column headers. " "Specify this option to assume no headers.", ) parser.add_argument( "--debug", action="store_true", help="Verbose debugging output.", ) progargs = parser.parse_args() kwargs = { "filenames": progargs.filenames, "input_dialect": progargs.inputdialect, "output_dialect": progargs.outputdialect, "debug": progargs.debug, "headers": not progargs.noheaders, } if progargs.outfile == '-': log.info("Writing to stdout") merge_csv(outfile=sys.stdout, **kwargs) else: log.info("Writing to " + repr(progargs.outfile)) with open(progargs.outfile, 'w') as outfile: # noinspection PyTypeChecker merge_csv(outfile=outfile, **kwargs)
def meta_main() -> None: """ Command-line process for ``camcops_server_meta`` tool. """ parser = argparse.ArgumentParser( description="Run commands across multiple CamCOPS databases") parser.add_argument('cc_command', type=str, help="Main command to pass to CamCOPS") parser.add_argument('--filespecs', nargs='+', required=True, help="List of CamCOPS config files (wildcards OK)") parser.add_argument( '--ccargs', nargs='*', help="List of CamCOPS arguments, to which '--' will be prefixed") parser.add_argument('--python', default=sys.executable, help="Python interpreter (default: {})".format( sys.executable)) parser.add_argument( '--camcops', default=DEFAULT_CAMCOPS, help="CamCOPS server executable (default: {})".format(DEFAULT_CAMCOPS)) parser.add_argument('-d', '--dummyrun', action="store_true", help="Dummy run (show filenames only)") parser.add_argument('-v', '--verbose', action="store_true", help="Verbose") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) log.debug("Arguments: {}", args) # Delayed import so --help doesn't take ages from camcops_server.camcops_server import main as camcops_main # delayed import # noqa did_something = False # old_sys_argv = sys.argv.copy() for filespec in args.filespecs: for filename in glob.glob(filespec): did_something = True log.info("Processing: {}", filename) sys.argv = ([ 'camcops_server', # dummy argv[0] args.cc_command, "--config", filename ] + ['--{}'.format(x) for x in args.ccargs or []]) log.debug("Executing command: {}", sys.argv) if args.dummyrun: continue camcops_main() # using the new sys.argv # subprocess.check_call(cmdargs) if not did_something: log.info("Nothing to do; no files found")
def main() -> NoReturn: """ Command-line processor. See ``--help`` for details. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) # noinspection PyTypeChecker parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( "input_file", help="Input PDF (which is not modified by this program)") parser.add_argument("output_file", help="Output PDF") parser.add_argument( "--slice_horiz", type=int, default=1, help="Slice the input PDF first into this many parts horizontally") parser.add_argument( "--slice_vert", type=int, default=1, help="Slice the input PDF first into this many parts vertically") parser.add_argument( "--longedge", action="store_true", help="Create PDF for long-edge duplex printing, not short edge") parser.add_argument("--overwrite", action="store_true", help="Allow overwriting of an existing output file") parser.add_argument( "--unittest", action="store_true", help="Run unit tests and exit (you must pass dummy values for " "input/output files to use these tests)") # ... because requiring dummy input/output filenames for unit testing # is less confusing for the majority of users than showing syntax in # which they are optional! args = parser.parse_args() if args.unittest: log.warning("Performing unit tests") # unittest.main() doesn't play nicely with argparse; they both # use sys.argv by default (and we end up with what looks like garbage # from the argparse help facility); but this works: unittest.main(argv=[sys.argv[0]]) sys.exit(EXIT_SUCCESS) success = convert_to_foldable( input_filename=os.path.abspath(args.input_file), output_filename=os.path.abspath(args.output_file), slice_horiz=args.slice_horiz, slice_vert=args.slice_vert, overwrite=args.overwrite, longedge=args.longedge) sys.exit(EXIT_SUCCESS if success else EXIT_FAILURE)
def main() -> None: """ Command-line entry point. """ description = "Manage users for the CRATE nlp_web server." parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter) arg_group = parser.add_mutually_exclusive_group() arg_group.add_argument("--adduser", nargs=2, metavar=("USERNAME", "PASSWORD"), help="Add a user and associated password.") arg_group.add_argument("--rmuser", nargs=1, metavar="USERNAME", help="Remove a user by specifying their username.") arg_group.add_argument("--changepw", nargs=2, metavar=("USERNAME", "PASSWORD"), help="Change a user's password.") args = parser.parse_args() main_only_quicksetup_rootlogger() if not args.adduser and not args.rmuser and not args.changepw: log.error( "One option required: '--adduser', '--rmuser' or '--changepw'.") return log.debug(f"Settings file: {SETTINGS_PATH}") log.debug(f"Users file: {USERS_FILENAME}") if args.rmuser: username = args.rmuser[0] proceed = input(f"Confirm remove user: {username} ? [yes/no] ") if proceed.lower() == "yes": rm_user(username) else: log.info("User remove aborted.") elif args.adduser: username = args.adduser[0] password = args.adduser[1] add_user(username, password) elif args.changepw: username = args.changepw[0] new_password = args.changepw[1] proceed = input( f"Confirm change password for user: {username} ? [yes/no] ") if proceed.lower() == "yes": change_password(username, new_password) else: log.info("Password change aborted.")
def main() -> None: """ Command-line processor. See ``--help`` for details. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) parser = argparse.ArgumentParser() parser.add_argument("directory", nargs="?", default=os.getcwd()) parser.add_argument("--reportevery", default=10000) args = parser.parse_args() log.info("Extensions in directory {!r}:", args.directory) print("\n".join( repr(x) for x in list_file_extensions(args.directory, reportevery=args.reportevery)))
def main() -> None: main_only_quicksetup_rootlogger() process("test1_equal_preferences_check_output_consistency.xlsx", "test_out1.xlsx") process("test2_trivial_perfect.xlsx", "test_out2.xlsx") process("test3_n60_two_equal_solutions.xlsx", "test_out3.xlsx") process("test4_n10_multiple_ties.xlsx", "test_out4.xlsx") process("test5_mean_vs_variance.xlsx", "test_out5.xlsx") process("test6_multiple_students_per_project.xlsx", "test_out6.xlsx") process("test7_eligibility.xlsx", "test_out7.xlsx") process("test8_tied_preferences.xlsx", "test_out8.xlsx", ["--allow_supervisor_preference_ties"]) process("test9_supervisor_constraints.xlsx", "test_out9.xlsx", ["--debug_model", "--no_shuffle"])
def main() -> None: """ Command-line entry point. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) gate_home = get_envvar_or_die(EnvVar.GATE_HOME) log.info(f"Note the unrealistic use of {DEMO_NLP_INPUT_TERMINATOR!r} as " f"an end-of-input marker. Don't use that for real!") check_call_verbose([ "java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{gate_home}/lib/*", f"-Dgate.home={gate_home}", "CrateGatePipeline", "--annotation", "Person", "--annotation", "Location", "--input_terminator", DEMO_NLP_INPUT_TERMINATOR, "--output_terminator", DEMO_NLP_OUTPUT_TERMINATOR, "--log_tag", ".", "-v", "-v", "--demo" ])
def main() -> None: """ Command-line entry point. """ main_only_quicksetup_rootlogger(level=logging.DEBUG) medex_home = get_envvar_or_die(EnvVar.MEDEX_HOME) check_call_verbose([ "java", "-classpath", f"{CrateDir.JAVA_CLASSES}:{medex_home}/bin:{medex_home}/lib/*", "CrateMedexPipeline", "--help", "-v", "-v", ])
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--make", action="store_true", help="Do things! Otherwise will just show its intent.") parser.add_argument("--destroy_first", action="store_true", help="Destroy all existing autodocs first") parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) make_autodoc(make=args.make, destroy_first=args.destroy_first)
def main() -> None: """ Command-line entry point. """ parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( "source_codes", nargs="+", type=int, help="Source codes to look up " "(along with their descendants if --descendants is specified)" ) parser.add_argument( "--descendants", action="store_true", help="Include descendants of the codes specified" ) parser.add_argument( "--concept", type=str, default=DEFAULT_CONCEPT, help="Athena OHDSI CONCEPT.csv TSV file including the source and " "destination vocabularies" ) parser.add_argument( "--concept_relationship", type=str, default=DEFAULT_CONCEPT_RELATIONSHIP, help="Athena OHDSI CONCEPT_RELATIONSHIP.csv TSV file " "including the source and destination vocabularies" ) parser.add_argument( "--src_vocabulary", type=str, default=AthenaVocabularyId.SNOMED, help="Source vocabulary" ) parser.add_argument( "--dest_vocabulary", type=str, default=AthenaVocabularyId.OPCS4, help="Destination vocabulary" ) args = parser.parse_args() main_only_quicksetup_rootlogger() print_equivalent_opcs_codes( source_codes=args.source_codes, source_vocabulary=args.src_vocabulary, destination_vocabulary=args.dest_vocabulary, concept_file=args.concept, concept_relationship_file=args.concept_relationship, with_descendants=args.descendants, )
def main() -> None: main_only_quicksetup_rootlogger(level=logging.DEBUG, with_thread_id=True) if PROFILE: pr = cProfile.Profile() pr.enable() test_two_antidepressant_episodes() pr.disable() s = io.StringIO() sortby = 'cumulative' # In Python 3.7: sortby = pstats.SortKey.CUMULATIVE ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats(50) # top 50 print(s.getvalue()) else: test_two_antidepressant_episodes()
def main() -> None: main_only_quicksetup_rootlogger(level=logging.DEBUG) # Define jobs possible_jobs = ["experiment_1", "experiment_1b", "experiment_2"] # Command-line arguments parser = argparse.ArgumentParser( description="Launch spread-modelling job on SLURM cluster", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("jobs", type=str, nargs="+", choices=possible_jobs + ["all"], help="Job code") args = parser.parse_args() # Launch jobs = possible_jobs if "all" in args.jobs else args.jobs for job in jobs: launch(job)
def main() -> None: parser = argparse.ArgumentParser( description="Parse a mailbox file from stdin to stdout, " "removing any attachments", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("input", type=str, help="Filename for input") parser.add_argument("output", type=str, help="Filename for output") parser.add_argument("--verbose", "-v", action="store_true", help="Be verbose") parser.add_argument("--report", type=int, default=100, help="Report every n messages") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) if os.path.exists(args.output): errmsg = f"Output file exists: {args.output}" log.critical(errmsg) raise ValueError(errmsg) log.info("Opening input file: {filename} ({size})", filename=args.input, size=bytes2human(os.path.getsize(args.input))) input_box = mailbox.mbox(args.input, create=False) log.info("Opening output file: {} (new file)", args.output) output_box = mailbox.mbox(args.output, create=True) log.info("Processing messages...") msg_count = 0 for message in input_box.itervalues(): msg_count += 1 if msg_count % args.report == 0: log.debug("Processing message {}", msg_count) processed_msg = clean_message(message, topmost=True) output_box.add(processed_msg) log.info("Done; processed {} messages.", msg_count) log.info("Output size: {}", bytes2human(os.path.getsize(args.output)))
def make_wsgi_app(global_config: Dict[Any, Any], **settings) -> Router: # Logging main_only_quicksetup_rootlogger() logging.getLogger('sqlalchemy').setLevel(logging.WARNING) # log.debug(f"global_config: {global_config!r}") # log.debug(f"settings: {settings!r}") # Database engine = engine_from_config(settings, NlpServerConfigKeys._SQLALCHEMY_PREFIX, **SQLALCHEMY_COMMON_OPTIONS) # ... add to config - pool_recycle is set to create new sessions every 7h sqla_url = get_safe_url_from_engine(engine) log.info(f"Using database {sqla_url!r}") dbsession.configure(bind=engine) Base.metadata.bind = engine # Pyramid config = Configurator(settings=settings) # Security policies authn_policy = AuthTktAuthenticationPolicy( settings[NlpServerConfigKeys.NLP_WEBSERVER_SECRET], secure=True, # only allow requests over HTTPS hashalg='sha512') authz_policy = ACLAuthorizationPolicy() config.set_authentication_policy(authn_policy) config.set_authorization_policy(authz_policy) # Compression config.add_tween("cardinal_pythonlib.pyramid.compression.CompressionTweenFactory") # noqa # Routes config.add_route('index', '/') config.scan('.views') # Create WSGI app return config.make_wsgi_app()
def main() -> None: """ Command-line processor. See ``--help`` for details. """ parser = ArgumentParser(description="Remove duplicate files") parser.add_argument( "directory", nargs="+", help="Files and/or directories to check and remove duplicates from.") parser.add_argument("--recursive", action="store_true", help="Recurse through any directories found") parser.add_argument("--dummy_run", action="store_true", help="Dummy run only; don't actually delete anything") parser.add_argument( "--run_repeatedly", type=int, help="Run the tool repeatedly with a pause of <run_repeatedly> " "seconds between runs. (For this to work well," "you should specify one or more DIRECTORIES in " "the 'filename' arguments, not files, and you will need the " "--recursive option.)") parser.add_argument("--verbose", action="store_true", help="Verbose output") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) while True: deduplicate(args.directory, recursive=args.recursive, dummy_run=args.dummy_run) if args.run_repeatedly is None: break log.info("Sleeping for {} s...", args.run_repeatedly) sleep(args.run_repeatedly)
""" import argparse import logging from cardinal_pythonlib.logs import main_only_quicksetup_rootlogger from crate_anon.nlp_manager import ( all_processors, regex_parser, regex_units, ) def test_all_regex_nlp(verbose: bool = False) -> None: """ Test all NLP-related regular expressions. """ regex_parser.test_all(verbose=verbose) # basic regexes regex_units.test_all(verbose=verbose) all_processors.test_all_processors(verbose=verbose) # ... tests all parser classes if __name__ == '__main__': main_only_quicksetup_rootlogger(level=logging.DEBUG) parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action="store_true", help="Verbose") args = parser.parse_args() test_all_regex_nlp(verbose=args.verbose)
def main() -> None: """ Creates an Alembic database migration for CamCOPS, by comparing the metadata (from Python) with the current database. Note special difficulty with "variant" types, such as ``Integer().with_variant(...)`` which are (2017-08-21, alembic==0.9.4) rendered as ``sa.Variant()`` only with a MySQL backend. - https://bitbucket.org/zzzeek/alembic/issues/433/variant-base-not-taken-into-account-when - https://bitbucket.org/zzzeek/alembic/issues/131/create-special-rendering-for-variant We deal with these via :func:`camcops_server.alembic.env.process_revision_directives` in ``env.py``. """ # noqa desc = ( "Create database revision. Note:\n" "- The config used will be that from the environment variable {} " "(currently {!r}).\n" "- Alembic compares (a) the current state of the DATABASE to (b) " "the state of the SQLAlchemy metadata (i.e. the CODE). It creates a " "migration file to change the database to match the code.\n" "- Accordingly, in the rare event of wanting to do a fresh start, you " "need an *empty* database.\n" "- More commonly, you want a database that is synced to a specific " "Alembic version (with the correct structure, and the correct version " "in the alembic_version table). If you have made manual changes, such " "that the actual database structure doesn't match the structure that " "Alembic expects based on that version, there's likely to be " "trouble.\n".format( ENVVAR_CONFIG_FILE, environ.get(ENVVAR_CONFIG_FILE, None) ) ) wrapped = "\n\n".join( textwrap.fill(x, width=79, initial_indent="", subsequent_indent=" ") for x in desc.splitlines() ) # noinspection PyTypeChecker parser = ArgumentParser( description=wrapped, formatter_class=RawDescriptionHelpFormatter ) parser.add_argument("message", help="Revision message") parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO ) # ... hmpf; ignored (always debug); possible Alembic forces this. # Check the existing database version is OK. config = get_default_config_from_os_env() config.assert_database_ok() # Then, if OK, create an upgrade. create_database_migration_numbered_style( alembic_ini_file=ALEMBIC_INI_FILE, alembic_versions_dir=ALEMBIC_VERSIONS_DIR, message=args.message, n_sequence_chars=N_SEQUENCE_CHARS, ) print( r""" Now: - Check the new migration file. - Check in particular for incorrectly dialect-specific stuff, e.g. with grep "mysql\." *.py ... should only show "sa.SOMETYPE().with_variant(mysql.MYSQLTYPE..." - and grep "sa\.Variant" *.py ... suggests an error that should be "Sometype().with_variant(...)"; see source here. """ )
def main(): """ Command-line entry point. """ check_prerequisites() # noinspection PyTypeChecker parser = argparse.ArgumentParser( description=""" - Creates a Debian (.deb) and RPM (.rpm) distribution file for the CamCOPS server, for distribution under Linux. - In brief, the following sequence is followed as the package is built: * The CamCOPS server is packaged up from source using python setup.py sdist and zipped in a Debian-safe way. * The principle is that the Python package should do all the work, not the Debian framework. This also means that a user who elects to install via pip gets exactly the same functional file structure. * A Debian package is built, containing all the usual Debian goodies (man packages, the preinst/postinst/prerm/postrm files, cataloguing and control files, etc.). * The package is checked with Lintian. * An RPM is built from the .deb package. - The user then installs the DEB or RPM file. In addition to installing standard things like man pages, this then: * attempts to stop supervisord for the duration of the installation (because that's the usual way to run a CamCOPS server); * creates a few standard directories (e.g. for CamCOPS configuration and lock files), including {LINUX_DEFAULT_CAMCOPS_CONFIG_DIR} {LINUX_DEFAULT_CAMCOPS_DIR} {LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR} {LINUX_DEFAULT_LOCK_DIR} * checks that Python is available on the system; * uses the system Python to create a Python virtual environment within {LINUX_DEFAULT_CAMCOPS_DIR}; * uses the virtual environment's "pip" command to install the distributed CamCOPS Python package within that virtual environment; ... which also appears to compile .py to .pyc files automatically; * creates master executable scripts (which call corresponding Python scripts): {DSTCONSOLEFILE} {DSTMETACONSOLEFILE} * sets some permissions (to default users such as "www-data" on Ubuntu, or "apache" on CentOS); * restarts supervisord. """.format( DSTCONSOLEFILE=DSTCONSOLEFILE, DSTMETACONSOLEFILE=DSTMETACONSOLEFILE, LINUX_DEFAULT_CAMCOPS_CONFIG_DIR=LINUX_DEFAULT_CAMCOPS_CONFIG_DIR, LINUX_DEFAULT_CAMCOPS_DIR=LINUX_DEFAULT_CAMCOPS_DIR, LINUX_DEFAULT_LOCK_DIR=LINUX_DEFAULT_LOCK_DIR, LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR= LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR, # noqa ), formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--verbose", "-v", action="store_true") args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) log.info("mainversion: {}", MAINVERSION) log.info("changedate: {}", CHANGEDATE) build_package()
run_cmd(["crate_django_manage", "help", "lookup_patient"], join(WEB_DIR, "_crate_django_manage_lookup_patient_help.txt")) run_cmd(["crate_django_manage", "help", "populate"], join(WEB_DIR, "_crate_django_manage_populate_help.txt")) run_cmd([ "crate_django_manage", "help", "resubmit_unprocessed_tasks" ], join( WEB_DIR, "_crate_django_manage_resubmit_unprocessed_tasks_help.txt")) # noqa run_cmd(["crate_django_manage", "help", "runcpserver"], join(WEB_DIR, "_crate_django_manage_runcpserver_help.txt")) run_cmd(["crate_django_manage", "help", "runserver"], join(WEB_DIR, "_crate_django_manage_runserver_help.txt")) run_cmd(["crate_django_manage", "help", "test_email"], join(WEB_DIR, "_crate_django_manage_test_email_help.txt")) run_cmd(["crate_launch_celery", "--help"], join(WEB_DIR, "_crate_launch_celery_help.txt")) run_cmd(["crate_launch_django_server", "--help"], join(WEB_DIR, "_crate_launch_django_server_help.txt")) run_cmd(["crate_print_demo_crateweb_config"], join(WEB_DIR, "_specimen_web_config.py")) log.warning("Skipping crate_windows_service_help.txt (requires Windows)") log.info("Done.") if __name__ == "__main__": main_only_quicksetup_rootlogger() main()
def main() -> None: """ Command-line processor. See ``--help`` for details. """ main_only_quicksetup_rootlogger() wdcd_suffix = "_with_drop_create_database" timeformat = "%Y%m%dT%H%M%S" parser = argparse.ArgumentParser( description=f""" Back up a specific MySQL database. The resulting filename has the format '<DATABASENAME>_<DATETIME><SUFFIX>.sql', where <DATETIME> is of the ISO-8601 format {timeformat!r} and <SUFFIX> is either blank or {wdcd_suffix!r}. A typical filename is therefore 'mydb_20190415T205524.sql'. """, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("databases", nargs="+", help="Database(s) to back up") parser.add_argument("--max_allowed_packet", default="1GB", help="Maximum size of buffer") parser.add_argument("--mysqldump", default="mysqldump", help="mysqldump executable") parser.add_argument("--username", default="root", help="MySQL user") parser.add_argument( "--password", help="MySQL password (AVOID THIS OPTION IF POSSIBLE; VERY INSECURE; " "VISIBLE TO OTHER PROCESSES; if you don't use it, you'll be " "prompted for the password)") parser.add_argument( "--with_drop_create_database", action="store_true", help="Include DROP DATABASE and CREATE DATABASE commands, and append " "a suffix to the output filename as above") parser.add_argument( "--output_dir", type=str, help="Output directory (if not specified, current directory will be " "used)") parser.add_argument("--verbose", action="store_true", help="Verbose output") args = parser.parse_args() output_dir = args.output_dir or os.getcwd() os.chdir(output_dir) password = args.password or getpass.getpass( prompt=f"MySQL password for user {args.username}: ") output_files = [] # type: List[str] if args.with_drop_create_database: log.info("Note that the DROP DATABASE commands will look like they're " "commented out, but they're not: " "https://dba.stackexchange.com/questions/59294/") suffix = wdcd_suffix else: suffix = "" for db in args.databases: now = datetime.datetime.now().strftime(timeformat) outfilename = f"{db}_{now}{suffix}.sql" display_args = cmdargs( mysqldump=args.mysqldump, username=args.username, password=password, database=db, verbose=args.verbose, with_drop_create_database=args.with_drop_create_database, max_allowed_packet=args.max_allowed_packet, hide_password=True) actual_args = cmdargs( mysqldump=args.mysqldump, username=args.username, password=password, database=db, verbose=args.verbose, with_drop_create_database=args.with_drop_create_database, max_allowed_packet=args.max_allowed_packet, hide_password=False) log.info("Executing: " + repr(display_args)) log.info("Output file: " + repr(outfilename)) try: with open(outfilename, "w") as f: subprocess.check_call(actual_args, stdout=f) except subprocess.CalledProcessError: os.remove(outfilename) log.critical("Failed!") sys.exit(1) output_files.append(outfilename) log.info("Done. See:\n" + "\n".join(" " + x for x in output_files)) if args.with_drop_create_database: log.info("To restore: mysql -u USER -p < BACKUP.sql") else: log.info("To restore: mysql -u USER -p DATABASE < BACKUP.sql")
def main() -> None: """ Command-line entry point. """ # Parser parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument("--cachepath", type=str, default=DEFAULT_CACHE_PATH, help="Cache path for ChEBI files") parser.add_argument("--verbose", action="store_true", help="Be verbose") subparsers = parser.add_subparsers(title="subcommands", description="Valid subcommands", dest="command") subparsers.required = True def add_exact(p: argparse.ArgumentParser) -> None: p.add_argument("--exact_search", dest="exact_search", action="store_true", help="Search using exact term") p.add_argument("--inexact_search", dest="exact_search", action="store_false", help="Search allowing inexact matches") p.set_defaults(exact_search=DEFAULT_EXACT_SEARCH) p.add_argument("--exact_match", dest="exact_match", action="store_true", help="Return results for exact term only") p.add_argument("--inexact_match", dest="exact_match", action="store_false", help="Return results allowing inexact matches") p.set_defaults(exact_match=DEFAULT_EXACT_MATCH) def add_entities(p: argparse.ArgumentParser) -> None: p.add_argument("entity", type=str, nargs="+", help="Entity or entities to search for") # ------------------------------------------------------------------------- # Test # ------------------------------------------------------------------------- parser_test = subparsers.add_parser("test", help="Run some simple tests") parser_test.set_defaults(func=lambda args: testfunc1()) # ------------------------------------------------------------------------- # Search # ------------------------------------------------------------------------- parser_search = subparsers.add_parser( "search", help="Search for an entity in the ChEBI database") add_entities(parser_search) add_exact(parser_search) parser_search.set_defaults(func=lambda args: search_and_list_multiple( search_terms=args.entity, exact_search=args.exact_search, exact_match=args.exact_match, )) # ------------------------------------------------------------------------- # Describe # ------------------------------------------------------------------------- parser_describe = subparsers.add_parser( "describe", help="Describe an entity/entities in the ChEBI database") add_entities(parser_describe) add_exact(parser_describe) parser_describe.set_defaults( func=lambda args: search_and_describe_multiple( # noqa search_terms=args.entity, exact_search=args.exact_search, exact_match=args.exact_match, )) # ------------------------------------------------------------------------- # Ancestors # ------------------------------------------------------------------------- parser_ancestors = subparsers.add_parser( "ancestors", help="Show ancestors of an entity/entities in the ChEBI database") add_entities(parser_ancestors) parser_ancestors.add_argument( "--relationships", type=str, nargs="+", default=DEFAULT_ANCESTOR_RELATIONSHIPS, help="Relationship types that define an ancestor") parser_ancestors.add_argument( "--max_generations", type=int, default=None, help="Number of generations to search, or None for unlimited") parser_ancestors.set_defaults(func=lambda args: report_ancestors_multiple( entity_names=args.entity, relationships=args.relationships, max_generations=args.max_generations, )) # ------------------------------------------------------------------------- # Categorize # ------------------------------------------------------------------------- parser_categorize = subparsers.add_parser( "categorize", help="Categorize a list of drugs.") parser_categorize.add_argument( "--entities", type=str, required=True, help="Input file, one entity (e.g. drug) name per line.") parser_categorize.add_argument( "--categories", type=str, required=True, help="Name of file containing categories, one per line " "(earlier categories preferred to later).") parser_categorize.add_argument( "--entity_synonyms", type=str, default=None, help="Name of CSV file (with optional # comments) containing synonyms " "in the format 'entity_from, entity_to'") parser_categorize.add_argument( "--category_synonyms", type=str, default=None, help="Name of CSV file (with optional # comments) containing synonyms " "in the format 'category_from, category_to'. The translation is " "applied to ChEBI categories before matching. For example you " "can map 'EC 3.1.1.7 (acetylcholinesterase) inhibitor' to " "'acetylcholinesterase inhibitor' and then use only " "'acetylcholinesterase inhibitor' in your category file.") parser_categorize.add_argument( "--manual_categories", type=str, default=None, help="Name of CSV file (with optional # comments) containing manual " "categorizations in the format 'entity, category'") parser_categorize.add_argument("--results", type=str, required=True, help="Output CSV file.") parser_categorize.add_argument( "--relationships", type=str, nargs="+", default=DEFAULT_ANCESTOR_RELATIONSHIPS, help="Relationship types that define an ancestor") parser_categorize.set_defaults(func=lambda args: categorize_from_file( entity_filename=args.entities, results_filename=args.results, category_filename=args.categories, entity_synonyms_filename=args.entity_synonyms, category_synonyms_filename=args.category_synonyms, manual_categories_filename=args.manual_categories, relationships=args.relationships, )) # ------------------------------------------------------------------------- # Parse and run # ------------------------------------------------------------------------- cmdargs = parser.parse_args() # Logging main_only_quicksetup_rootlogger( level=logging.DEBUG if cmdargs.verbose else logging.INFO) log.debug(f"ChEBI lookup from cardinal_pythonlib=={VERSION_STRING}") # Caching log.debug(f"Using cache path: {cmdargs.cachepath}") set_download_cache_path(cmdargs.cachepath) # Do something useful cmdargs.func(cmdargs)
def main() -> None: """ Command-line handler for the ``pause_process_by_disk_space`` tool. Use the ``--help`` option for help. """ parser = ArgumentParser( description="Pauses and resumes a process by disk space; LINUX ONLY." ) parser.add_argument( "process_id", type=int, help="Process ID." ) parser.add_argument( "--path", required=True, help="Path to check free space for (e.g. '/')" ) parser.add_argument( "--pause_when_free_below", type=str, required=True, help="Pause process when free disk space below this value (in bytes " "or as e.g. '50G')" ) parser.add_argument( "--resume_when_free_above", type=str, required=True, help="Resume process when free disk space above this value (in bytes " "or as e.g. '70G')" ) parser.add_argument( "--check_every", type=int, required=True, help="Check every n seconds (where this is n)" ) parser.add_argument( "--verbose", action="store_true", help="Verbose output" ) args = parser.parse_args() main_only_quicksetup_rootlogger( level=logging.DEBUG if args.verbose else logging.INFO) minimum = human2bytes(args.pause_when_free_below) maximum = human2bytes(args.resume_when_free_above) path = args.path process_id = args.process_id period = args.check_every pause_args = ["kill", "-STOP", str(process_id)] resume_args = ["kill", "-CONT", str(process_id)] assert minimum < maximum, "Minimum must be less than maximum" log.info( f"Starting: controlling process {process_id}; " f"checking disk space every {period} s; " f"will pause when free space on {path} " f"is less than {sizeof_fmt(minimum)} and " f"resume when free space is at least {sizeof_fmt(maximum)}; " f"pause command will be {pause_args}; " f"resume command will be {resume_args}." ) log.debug("Presuming that the process is RUNNING to begin with.") paused = False while True: if not is_running(process_id): log.info("Process {} is no longer running", process_id) sys.exit(0) space = shutil.disk_usage(path).free log.debug("Disk space on {} is {}", path, sizeof_fmt(space)) if space < minimum and not paused: log.info("Disk space down to {}: pausing process {}", sizeof_fmt(space), process_id) subprocess.check_call(pause_args) paused = True elif space >= maximum and paused: log.info("Disk space up to {}: resuming process {}", sizeof_fmt(space), process_id) subprocess.check_call(resume_args) paused = False log.debug("Sleeping for {} seconds...", period) sleep(period)
"plain/class member function of same name (unless " "namespace is manually given)") test(t.no_params_dogpile_default_fkg(), False) t2 = SecondTestClass() test(t.dogpile_default_test_2(), True) test(t.dogpile_default_test_2(), False) try: test(t2.dogpile_default_test_2(), True) log.info("dogpile.cache default FKG correctly distinguishing between " "member functions of two different classes with same name") except AssertionError: log.warning("Known dogpile.cache default FKG problem - conflates " "member functions of two different classes where " "functions have same name (unless namespace is manually " "given)") test(t2.dogpile_default_test_2(), False) log.info("Success!") # TEST THIS WITH: # python -m cardinal_pythonlib.dogpile_cache if __name__ == '__main__': level = logging.DEBUG if TESTING_VERBOSE else logging.INFO if TESTING_USE_PRETTY_LOGS: main_only_quicksetup_rootlogger(level=level) else: logging.basicConfig(level=level) unit_test_cache()
def main() -> None: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('mdbfile', type=str, help="Microsoft Access .MDB file to read") parser.add_argument( '--schemaonly', action='store_true', help="Print schema SQL only (otherwise: will create MySQL database and" " write schema and data)") parser.add_argument( '--host', type=str, default='127.0.0.1', # not "localhost" help="MySQL hostname or IP address") parser.add_argument('--port', type=int, default=3306, help="MySQL port number") parser.add_argument('--user', type=str, default='root', help="MySQL username") parser.add_argument('--password', type=str, action=PasswordPromptAction, help="MySQL password") parser.add_argument('--mysqldb', type=str, help="MySQL database to use") parser.add_argument('--create', action="store_true", help="Create database?") args = parser.parse_args() if not args.schemaonly and not args.mysqldb: raise ValueError("Must specify --mysqldb, unless --schemaonly is used") main_only_quicksetup_rootlogger(level=logging.DEBUG) # ------------------------------------------------------------------------- log.info("Getting list of tables") # ------------------------------------------------------------------------- tablecmd = ["mdb-tables", "-1", args.mdbfile] # ... -1: one per line (or table names with spaces will cause confusion) tables = get_command_output(tablecmd).splitlines() tables.sort(key=lambda s: s.lower()) log.info("Tables: {}".format(tables)) # ------------------------------------------------------------------------- log.info("Fetching schema definition") # ------------------------------------------------------------------------- schemacmd = [ "mdb-schema", # from Ubuntu package mdbtools args.mdbfile, # database "mysql", # backend "--no-drop-table", # don't issue DROP TABLE statements (default) "--not-null", # issue NOT NULL constraints (default) "--no-default-values", # don't issue DEFAULT values (default) "--indexes", # export INDEX statements (default) "--relations", # export foreign key constraints (default) ] schemasyntax = get_command_output(schemacmd) # ------------------------------------------------------------------------- log.info("Converting any schema syntax oddities to MySQL syntax") # ------------------------------------------------------------------------- # JAN 2013: Since my previous script, mdb-schema's mysql dialect has got # much better. So, not much to do here. # "COMMENT ON COLUMN" produced by mdb-schema and rejected by MySQL: schemasyntax = re.sub("^COMMENT ON COLUMN.*$", "", schemasyntax, 0, re.MULTILINE) log.info("Schema syntax:") print(schemasyntax) # ------------------------------------------------------------------------- # Done? # ------------------------------------------------------------------------- if args.schemaonly: return # ------------------------------------------------------------------------- log.info("Creating new database") # ------------------------------------------------------------------------- createmysqldbcmd = [ "mysqladmin", "create", args.mysqldb, "--host={}".format(args.host), "--port={}".format(args.port), "--user={}".format(args.user), "--password={}".format(args.password) ] # We could omit the actual password and the user would be prompted, but we # need to send it this way later (see below), so this is not a huge # additional security weakness! # Linux/MySQL helpfully obscures the password in the "ps" list. log.info(get_command_output(createmysqldbcmd)) # ------------------------------------------------------------------------- log.info("Sending schema command to MySQL") # ------------------------------------------------------------------------- mysqlcmd = [ "mysql", "--host={}".format(args.host), "--port={}".format(args.port), "--database={}".format(args.mysqldb), "--user={}".format(args.user), "--password={}".format(args.password) ] # Regrettably, we need the password here, as stdin will come from a pipe print(get_pipe_series_output([mysqlcmd], schemasyntax)) # ------------------------------------------------------------------------- log.info("Copying data to MySQL") # ------------------------------------------------------------------------- # For the data, we won't store the intermediate stuff in Python's memory, # 'cos it's vast; I had one odd single-character mutation # from "TimeInSession_ms" to "TimeInSession_mc" at row 326444 (perhaps # therefore 37Mb or so into a long string). # And I was trying to export ~1m records in that table alone. # We'll use pipes instead and let the OS deal with the memory management. # ... BUT (Jan 2013): now mdb-tools is better, text-processing not # necessary - can use temporary disk file # Turns out the bottleneck is the import to MySQL, not the export from MDB. # So see http://dev.mysql.com/doc/refman/5.5/en/optimizing-innodb-bulk-data-loading.html # noqa # The massive improvement is by disabling autocommit. (Example source # database is 208M; largest table here is 554M as a textfile; it has # 1,686,075 rows.) This improvement was from 20 Hz to the whole database # in a couple of minutes (~13 kHz). Subsequent export from MySQL: takes a # second or two to write whole DB (177M textfile). for t in tables: log.info("Processing table {}".format(t)) exportcmd = [ 'mdb-export', '-I', 'mysql', # -I backend: INSERT statements, not CSV # MySQL's DATETIME field has this format: "YYYY-MM-DD HH:mm:SS" # so we want this from the export: '-D', '%Y-%m-%d %H:%M:%S', # -D: date format # ... don't put any extra quotes around it. args.mdbfile, # database t # table ] with tempfile.NamedTemporaryFile( mode="wt", encoding=sys.getdefaultencoding()) as outfile: print("SET autocommit=0;", file=outfile) subprocess.call(exportcmd, stdout=outfile) print("\nCOMMIT;", file=outfile) outfile.flush() outfile.seek(0) subprocess.call(mysqlcmd, stdin=outfile) log.info("Finished.")