def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Creates new image (output) where probability map (probmap) is semi-transparently overlayed in blue on top of base image (image) Example Usage: createprobmapoverlay.py baseimage.png probmap.png overlay.png """.format(version=chmutil.__version__) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: return _convert_image(os.path.abspath(theargs.image), os.path.abspath(theargs.probmap), os.path.abspath(theargs.output), theargs) finally: logging.shutdown()
def test_setup_logging_defaults(self): mylogger = logging.getLogger('fooey') core.setup_logging(mylogger) self.assertEqual(mylogger.getEffectiveLevel(), logging.WARNING) clogger = logging.getLogger('chmutil.core') self.assertEqual(clogger.getEffectiveLevel(), logging.WARNING) clusterlogger = logging.getLogger('chmutil.cluster') self.assertEqual(clusterlogger.getEffectiveLevel(), logging.WARNING) core.setup_logging(mylogger, loglevel='DEBUG') self.assertEqual(mylogger.getEffectiveLevel(), logging.DEBUG) clogger = logging.getLogger('chmutil.core') self.assertEqual(clogger.getEffectiveLevel(), logging.DEBUG) clusterlogger = logging.getLogger('chmutil.cluster') self.assertEqual(clusterlogger.getEffectiveLevel(), logging.DEBUG) core.setup_logging(mylogger, loglevel='INFO') self.assertEqual(mylogger.getEffectiveLevel(), logging.INFO) core.setup_logging(mylogger, loglevel='ERROR') self.assertEqual(mylogger.getEffectiveLevel(), logging.ERROR) core.setup_logging(mylogger, loglevel='CRITICAL') self.assertEqual(mylogger.getEffectiveLevel(), logging.CRITICAL)
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Runs Merge tiles for <taskid> specified on command line. Example Usage: mergetilerunner.py 1 /foo/chmjob --scratchdir /scratch """.format(version=chmutil.__version__) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: return _run_merge_job(theargs) finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} WARNING: THIS SCRIPT HAS NOT BEEN TESTED AND MAY NOT WORK Creates mrc stack (output) by extracting random tiles from images in (imagedir) Example Usage: createtrainingmrcstack.py ./myimages 5 ./result.foo """.format(version=chmutil.__version__) sys.stderr.write('\nTHIS PROGRAM IS AN ALPHA IMPLEMENTATION ' 'AND MAY NOT WORK\n\n') theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: sys.stderr.write('\nTHIS PROGRAM IS AN ALPHA IMPLEMENTATION ' 'AND MAY NOT WORK\n\n') return _create_mrc_stack(os.path.abspath(theargs.imagedir), theargs.numtiles, os.path.abspath(theargs.output), theargs) finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Given an image, code will convert that image to grayscale, then optionally autolevel, blur, and downsample the image saving it in png format. By default only the conversion ot grayscale is performed. Please note, regardless of order of optional flags the operations are performed in this order: 1. Grayscale 2. Equalize 3. AutoContrast 4. GaussianBlur 5. Downsample 6. Generate Tiles Example Usage: createchmimage.py someimage.tif someimage.png """.format(version=chmutil.__version__) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: return _convert_image(os.path.abspath(theargs.image), os.path.abspath(theargs.output), theargs) finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Merges set of image tiles in <imagedir> directory writing out a single merged image to <output> file Example Usage: mergetiles.py ./histeqimages merged_img.png """.format(version=chmutil.__version__) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: logger.debug('Setting Image.MAX_IMAGE_PIXELS to ' + str(theargs.maxpixels)) Image.MAX_IMAGE_PIXELS = theargs.maxpixels return _merge_image_tiles(os.path.abspath(theargs.imagedir), os.path.abspath(theargs.output), theargs.suffix) except Exception: logger.exception('Caught exception') return 2 finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Runs CHM for batched <taskid> specified on command. Normally this tool is invoked by a scheduler (SGE, SLURM, etc..) but can be run directly. The batched task that is run is determined by looking for the [<taskid>] entry in {batchchm} configuration file in the <jobdir>. The actual CHM tasks that will be run are in a comma delimited list in the {taskid} field under the [<taskid>] entry which correspond to tasks in {basechm} configuration file. The exit code of this tool will be 0 upon success or a value greater then 0 if any of the CHM tasks fails. Example of {batchchm} configuration: [1] taskids = 2 [2] taskids = 3 Example of task in {basechm}: [1] inputimage = foo.png args = -t 1,1 -t 1,2 -t 1,3 outputimage = tiles/foo.png/001.foo.png Example Usage: chmrunner.py 1 /foo/chmjob --scratchdir /scratch """.format(version=chmutil.__version__, taskid=CHMJobCreator.BCONFIG_TASK_ID, batchchm=CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME, basechm=CHMJobCreator.CONFIG_FILE_NAME) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, log_format=LOG_FORMAT, loglevel=theargs.loglevel) try: return _run_chm_job(theargs) finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Creates job scripts to run CHM on images in <images> directory using model specified in <model> directory. The generated scripts are put in <outdir>. CHM requires TWO phases of processing. In the FIRST phase CHM tasks are run which work on tiles of each image in the input. This is done to parallelize the processing as well as reduce the memory footprint of CHM which gets huge on tiles larger then 1000x1000. For example tiles of 500x500 easily use 4 to 6 gigabytes of ram. These tiles are stored on the filesystem under <outdir>/{rundir}/{tiles}/<image.png> directories described below. In the SECOND phase merge tasks are run which combine the tiles into what are known as probability maps. Probability maps are simply greyscale 8-bit images (values 0-255) of the same size as in the input images where the intensity of the pixel correlates to the probability that it belongs to the feature trained for in the trained model. The probability maps are stored in <outdir>/{rundir}/{probmaps} directory described below. Here is a breakdown of the following directories created under the <outdir>: {config} -- Configuration containing CHM tasks. At the top of this file are some are options common to all CHM tasks denoted by the header [DEFAULT] Following the [DEFAULT] section are options for each CHM task which are delimited by a [#] where # is a number starting at 1. Example default: [DEFAULT] chmutilversion = {version} images = /home/foo/images chmbin = /foo/chm.img model = /home/foo/trainedmodel tilesperjob = 50 tilesize = 512x512 overlapsize = 0x0 disablehisteqimages = False jobspernode = 1 cluster = rocce Example task: [1] inputimage = someimage.png args = -t 1,1 -t 1,2 -t 1,3 outputimage = someimage.png/001.someimage.png {mergeconfig} -- Configuration containing merge tasks runjobs.<cluster> -- Cluster submit script {rundir}/ -- Directory containing output of job {rundir}/{tiles} -- Directory containing a directory for every image where the intermediate tile images can be written {rundir}/{stdout} -- Directory containing output from CHM tasks {rundir}/{mergestdout} -- Directory containing output from merge tasks {rundir}/{tmp} -- Directory used to hold temporary CHM outputs {rundir}/{probmaps} -- Directory containing finished probability map images generated from merge phase {rundir}/{overlaymaps} -- Directory containing overlay images where input images are overlayed with probability maps. This is generated in merge phase {rundir}/<image.png> -- Directories containing image tiles from individual CHM tasks Example Usage: createchmjob.py ./images ./model ./mychmjob Once job is created invoke checkchmjob.py for job submission. """.format(version=chmutil.__version__, config=CHMJobCreator.CONFIG_FILE_NAME, mergeconfig=CHMJobCreator.MERGE_CONFIG_FILE_NAME, rundir=CHMJobCreator.RUN_DIR, stdout=CHMJobCreator.STDOUT_DIR, mergestdout=CHMJobCreator.MERGE_STDOUT_DIR, tmp=CHMJobCreator.TMP_DIR, probmaps=CHMJobCreator.PROBMAPS_DIR, overlaymaps=CHMJobCreator.OVERLAYMAPS_DIR, tiles=CHMJobCreator.TILES_DIR) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, loglevel=theargs.loglevel) try: theargs.rawargs = ' '.join(arglist) return _create_chm_job(theargs) finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Creates script to run CHM train on training data located in <images> and <labels> directory. Putting the trained model under <outdir>/{model}/ The generated script is put in <outdir>. Here is a breakdown of the following directories and files created under the <outdir>: {readme} -- Readme file containing arguments passed to this script and definitions of files and directories {runtrain}<cluster> -- Cluster submit script {stdout}/ -- Directory containing output from CHM train task {tmp}/ -- Directory used to hold temporary CHM train outputs. Once processing is complete this directory is renamed to {model}/ {model}/ -- Directory appears after proccessing completes and contained trained model. Example Usage: createchmtrainjob.py can take data in two formats, default and IMOD extraction mode. DEFAULT MODE: In default mode the training images are already put into two directories. Here is example usage of that mode: createchmtrainjob.py ./images ./labels ./run --cluster rocce IMOD EXTRACTION MODE: If IMOD is installed this script can extract properly structured data from .mrc and .mod files. Instead of passing directories to <images> and <labels> just pass an mrc and mod file respectively. Example: createchmtrainjob.py images.mrc labels.mod ./run --cluster rocce In the above mode the images.mrc must contain the training images and the labels.mod must have the contours for that images.mrc file. createchmtrainjob.py will take all the images from images.mrc and put them in <outdir>/{images} directory and extract all labels from labels.mod and put them in <outdir>/{labels}/ directory giving them the name x.###.png. createchmtrainjob.py does this by invoking mrc2tif -p on the images.mrc and by invoking imodmop -mask 1 on the labels.mod file followed by an mrc2tif -p on the mrc file created by imodmop. """.format(version=chmutil.__version__, stdout=STDOUT_DIR, tmp=TMP_DIR, model=MODEL_DIR, readme=README_FILE, runtrain=RUNTRAIN, images=IMAGES_DIR, labels=LABELS_DIR) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, loglevel=theargs.loglevel) try: theargs.rawargs = ' '.join(arglist) _create_directories_and_readme(theargs.outdir, theargs.rawargs) theargs.images, theargs.labels = _convert_mod_mrc_files(theargs) submit_cmd = _create_submit_script(theargs) sys.stdout.write('\n' + submit_cmd + '\n\n') return 0 finally: logging.shutdown()
def main(arglist): """Main function :param arglist: Should be set to sys.argv which is list of arguments passed on commandline including script being run as arg 0 :returns: exit code. 0 is success otherwise failure """ desc = """ Version {version} Examines a CHM job generated by createchmjob.py to see if any tasks still need to be run. When running in default mode this tool ONLY outputs a job summary. TO update configuration to submit processing be sure to add {submit} flag. NOTE: CHM requires TWO phases of processing. For more information on these phases please run: createchmjob.py --help for more information. This tool examines BOTH phases of processing. For the FIRST phase this tool examines the <jobdir>/{run_dir}/{tiles}/<image dirs> directories and verifies image files exist for every tile listed in the <jobdir>/{chmconfig} config file. For the SECOND phase, this tool examines the <jobdir>/{run_dir}/{probmaps} directory and verifies existance of final probability maps for each input image. NOTE: It is assumed no active tasks are running on this CHM job. Example usage default: checkchmjob.py /foo/somechmjob chmutil version: {version} Tiles: 500x500 with 20x20 overlap Disable histogram equalization in CHM: True Jobs: 50 tiles per job, 1 job per node Trained CHM model: /foo/somemodel CHM binary: /foo/chm.img CHM tasks: 4% complete (960 of 23,456 completed) Merge tasks: 0% complete (0 of 1,234 completed) """.format( version=chmutil.__version__, run_dir=CHMJobCreator.RUN_DIR, chmconfig=CHMJobCreator.CONFIG_FILE_NAME, batchchm=CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME, batchmerge=CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME, probmaps=CHMJobCreator.PROBMAPS_DIR, tiles=CHMJobCreator.TILES_DIR, submit=SUBMIT_FLAG, detailed=DETAILED_FLAG) theargs = _parse_arguments(desc, arglist[1:]) theargs.program = arglist[0] theargs.version = chmutil.__version__ core.setup_logging(logger, loglevel=theargs.loglevel) try: return _check_chm_job(theargs) finally: logging.shutdown()