def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Creates new image (output)
where probability map (probmap) is semi-transparently
overlayed in blue on top of base image (image)
Example Usage:
createprobmapoverlay.py baseimage.png probmap.png overlay.png
""".format(version=chmutil.__version__)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger, log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
return _convert_image(os.path.abspath(theargs.image),
os.path.abspath(theargs.probmap),
os.path.abspath(theargs.output),
theargs)
finally:
logging.shutdown()
def test_setup_logging_defaults(self):
mylogger = logging.getLogger('fooey')
core.setup_logging(mylogger)
self.assertEqual(mylogger.getEffectiveLevel(), logging.WARNING)
clogger = logging.getLogger('chmutil.core')
self.assertEqual(clogger.getEffectiveLevel(), logging.WARNING)
clusterlogger = logging.getLogger('chmutil.cluster')
self.assertEqual(clusterlogger.getEffectiveLevel(), logging.WARNING)
core.setup_logging(mylogger, loglevel='DEBUG')
self.assertEqual(mylogger.getEffectiveLevel(), logging.DEBUG)
clogger = logging.getLogger('chmutil.core')
self.assertEqual(clogger.getEffectiveLevel(), logging.DEBUG)
clusterlogger = logging.getLogger('chmutil.cluster')
self.assertEqual(clusterlogger.getEffectiveLevel(), logging.DEBUG)
core.setup_logging(mylogger, loglevel='INFO')
self.assertEqual(mylogger.getEffectiveLevel(), logging.INFO)
core.setup_logging(mylogger, loglevel='ERROR')
self.assertEqual(mylogger.getEffectiveLevel(), logging.ERROR)
core.setup_logging(mylogger, loglevel='CRITICAL')
self.assertEqual(mylogger.getEffectiveLevel(), logging.CRITICAL)
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Runs Merge tiles for <taskid> specified on command
line.
Example Usage:
mergetilerunner.py 1 /foo/chmjob --scratchdir /scratch
""".format(version=chmutil.__version__)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger,
log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
return _run_merge_job(theargs)
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
WARNING: THIS SCRIPT HAS NOT BEEN TESTED AND MAY NOT WORK
Creates mrc stack (output) by extracting random
tiles from images in (imagedir)
Example Usage:
createtrainingmrcstack.py ./myimages 5 ./result.foo
""".format(version=chmutil.__version__)
sys.stderr.write('\nTHIS PROGRAM IS AN ALPHA IMPLEMENTATION '
'AND MAY NOT WORK\n\n')
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger,
log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
sys.stderr.write('\nTHIS PROGRAM IS AN ALPHA IMPLEMENTATION '
'AND MAY NOT WORK\n\n')
return _create_mrc_stack(os.path.abspath(theargs.imagedir),
theargs.numtiles,
os.path.abspath(theargs.output), theargs)
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Given an image, code will convert that image to grayscale,
then optionally autolevel, blur, and downsample the image
saving it in png format.
By default only the conversion ot grayscale is performed.
Please note, regardless of order of optional flags the
operations are performed in this order:
1. Grayscale
2. Equalize
3. AutoContrast
4. GaussianBlur
5. Downsample
6. Generate Tiles
Example Usage:
createchmimage.py someimage.tif someimage.png
""".format(version=chmutil.__version__)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger, log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
return _convert_image(os.path.abspath(theargs.image),
os.path.abspath(theargs.output),
theargs)
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Merges set of image tiles in <imagedir> directory
writing out a single merged image to <output> file
Example Usage:
mergetiles.py ./histeqimages merged_img.png
""".format(version=chmutil.__version__)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger,
log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
logger.debug('Setting Image.MAX_IMAGE_PIXELS to ' +
str(theargs.maxpixels))
Image.MAX_IMAGE_PIXELS = theargs.maxpixels
return _merge_image_tiles(os.path.abspath(theargs.imagedir),
os.path.abspath(theargs.output),
theargs.suffix)
except Exception:
logger.exception('Caught exception')
return 2
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Runs CHM for batched <taskid> specified on command.
Normally this tool is invoked by a scheduler (SGE, SLURM, etc..)
but can be run directly.
The batched task that is run is determined by
looking for the [<taskid>] entry in
{batchchm} configuration
file in the <jobdir>.
The actual CHM tasks that will be run are in a comma
delimited list in the {taskid} field under the
[<taskid>] entry which correspond to tasks in
{basechm} configuration file.
The exit code of this tool will be 0 upon success or a value
greater then 0 if any of the CHM tasks fails.
Example of {batchchm} configuration:
[1]
taskids = 2
[2]
taskids = 3
Example of task in {basechm}:
[1]
inputimage = foo.png
args = -t 1,1 -t 1,2 -t 1,3
outputimage = tiles/foo.png/001.foo.png
Example Usage:
chmrunner.py 1 /foo/chmjob --scratchdir /scratch
""".format(version=chmutil.__version__,
taskid=CHMJobCreator.BCONFIG_TASK_ID,
batchchm=CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME,
basechm=CHMJobCreator.CONFIG_FILE_NAME)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger,
log_format=LOG_FORMAT,
loglevel=theargs.loglevel)
try:
return _run_chm_job(theargs)
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Creates job scripts to run CHM on images in <images> directory
using model specified in <model> directory. The generated scripts
are put in <outdir>.
CHM requires TWO phases of processing.
In the FIRST phase CHM tasks are run which work on tiles of each
image in the input. This is done to parallelize the processing
as well as reduce the memory footprint of CHM which gets huge on
tiles larger then 1000x1000. For example tiles of 500x500 easily
use 4 to 6 gigabytes of ram. These tiles are stored on the
filesystem under <outdir>/{rundir}/{tiles}/<image.png>
directories described below.
In the SECOND phase merge tasks are run which combine the tiles
into what are known as probability maps. Probability maps are
simply greyscale 8-bit images (values 0-255)
of the same size as in the input images where the intensity of
the pixel correlates to the probability that it belongs to the
feature trained for in the trained model. The probability maps
are stored in <outdir>/{rundir}/{probmaps} directory described
below.
Here is a breakdown of the following directories
created under the <outdir>:
{config}
-- Configuration containing CHM tasks.
At the top of this file are some are options common
to all CHM tasks denoted by the header [DEFAULT]
Following the [DEFAULT] section are options for each
CHM task which are delimited by a [#] where # is a number
starting at 1.
Example default:
[DEFAULT]
chmutilversion = {version}
images = /home/foo/images
chmbin = /foo/chm.img
model = /home/foo/trainedmodel
tilesperjob = 50
tilesize = 512x512
overlapsize = 0x0
disablehisteqimages = False
jobspernode = 1
cluster = rocce
Example task:
[1]
inputimage = someimage.png
args = -t 1,1 -t 1,2 -t 1,3
outputimage = someimage.png/001.someimage.png
{mergeconfig}
-- Configuration containing merge tasks
runjobs.<cluster>
-- Cluster submit script
{rundir}/
-- Directory containing output of job
{rundir}/{tiles}
-- Directory containing a directory for every
image where the intermediate tile images can
be written
{rundir}/{stdout}
-- Directory containing output from CHM tasks
{rundir}/{mergestdout}
-- Directory containing output from merge tasks
{rundir}/{tmp}
-- Directory used to hold temporary CHM outputs
{rundir}/{probmaps}
-- Directory containing finished probability map images
generated from merge phase
{rundir}/{overlaymaps}
-- Directory containing overlay images where input images
are overlayed with probability maps. This is generated in
merge phase
{rundir}/<image.png>
-- Directories containing image tiles from individual
CHM tasks
Example Usage:
createchmjob.py ./images ./model ./mychmjob
Once job is created invoke checkchmjob.py for job submission.
""".format(version=chmutil.__version__,
config=CHMJobCreator.CONFIG_FILE_NAME,
mergeconfig=CHMJobCreator.MERGE_CONFIG_FILE_NAME,
rundir=CHMJobCreator.RUN_DIR,
stdout=CHMJobCreator.STDOUT_DIR,
mergestdout=CHMJobCreator.MERGE_STDOUT_DIR,
tmp=CHMJobCreator.TMP_DIR,
probmaps=CHMJobCreator.PROBMAPS_DIR,
overlaymaps=CHMJobCreator.OVERLAYMAPS_DIR,
tiles=CHMJobCreator.TILES_DIR)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger, loglevel=theargs.loglevel)
try:
theargs.rawargs = ' '.join(arglist)
return _create_chm_job(theargs)
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Creates script to run CHM train on training data located
in <images> and <labels> directory. Putting the trained
model under <outdir>/{model}/
The generated script is put in <outdir>.
Here is a breakdown of the following directories and files
created under the <outdir>:
{readme}
-- Readme file containing arguments passed to
this script and definitions of files and
directories
{runtrain}<cluster>
-- Cluster submit script
{stdout}/
-- Directory containing output from CHM train task
{tmp}/
-- Directory used to hold temporary CHM train outputs.
Once processing is complete this directory is
renamed to {model}/
{model}/
-- Directory appears after proccessing completes and
contained trained model.
Example Usage:
createchmtrainjob.py can take data in two formats, default
and IMOD extraction mode.
DEFAULT MODE:
In default mode the training images are already put into two
directories. Here is example usage of that mode:
createchmtrainjob.py ./images ./labels ./run --cluster rocce
IMOD EXTRACTION MODE:
If IMOD is installed this script can extract properly structured
data from .mrc and .mod files. Instead of passing directories to
<images> and <labels> just pass an mrc and mod file respectively.
Example:
createchmtrainjob.py images.mrc labels.mod ./run --cluster rocce
In the above mode the images.mrc must contain the training images
and the labels.mod must have the contours for that images.mrc
file. createchmtrainjob.py will take all the images from
images.mrc and put them in <outdir>/{images} directory and
extract all labels from labels.mod and put them in
<outdir>/{labels}/ directory giving them the name x.###.png.
createchmtrainjob.py does this by invoking mrc2tif -p on the
images.mrc and by invoking imodmop -mask 1 on the labels.mod
file followed by an mrc2tif -p on the mrc file created by
imodmop.
""".format(version=chmutil.__version__,
stdout=STDOUT_DIR,
tmp=TMP_DIR,
model=MODEL_DIR,
readme=README_FILE,
runtrain=RUNTRAIN,
images=IMAGES_DIR,
labels=LABELS_DIR)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger, loglevel=theargs.loglevel)
try:
theargs.rawargs = ' '.join(arglist)
_create_directories_and_readme(theargs.outdir, theargs.rawargs)
theargs.images, theargs.labels = _convert_mod_mrc_files(theargs)
submit_cmd = _create_submit_script(theargs)
sys.stdout.write('\n' + submit_cmd + '\n\n')
return 0
finally:
logging.shutdown()
def main(arglist):
"""Main function
:param arglist: Should be set to sys.argv which is list of arguments
passed on commandline including script being run as arg 0
:returns: exit code. 0 is success otherwise failure
"""
desc = """
Version {version}
Examines a CHM job generated by createchmjob.py to see
if any tasks still need to be run. When running in
default mode this tool ONLY outputs a job summary.
TO update configuration to submit processing be sure to
add {submit} flag.
NOTE: CHM requires TWO phases of processing.
For more information on these phases please run:
createchmjob.py --help
for more information.
This tool examines BOTH phases of processing.
For the FIRST phase this tool examines the
<jobdir>/{run_dir}/{tiles}/<image dirs> directories and
verifies image files exist for every tile listed in the
<jobdir>/{chmconfig} config file.
For the SECOND phase, this tool examines the
<jobdir>/{run_dir}/{probmaps} directory
and verifies existance of final probability maps for
each input image.
NOTE: It is assumed no active tasks are running on this CHM job.
Example usage default:
checkchmjob.py /foo/somechmjob
chmutil version: {version}
Tiles: 500x500 with 20x20 overlap
Disable histogram equalization in CHM: True
Jobs: 50 tiles per job, 1 job per node
Trained CHM model: /foo/somemodel
CHM binary: /foo/chm.img
CHM tasks: 4% complete (960 of 23,456 completed)
Merge tasks: 0% complete (0 of 1,234 completed)
""".format(
version=chmutil.__version__,
run_dir=CHMJobCreator.RUN_DIR,
chmconfig=CHMJobCreator.CONFIG_FILE_NAME,
batchchm=CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME,
batchmerge=CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME,
probmaps=CHMJobCreator.PROBMAPS_DIR,
tiles=CHMJobCreator.TILES_DIR,
submit=SUBMIT_FLAG,
detailed=DETAILED_FLAG)
theargs = _parse_arguments(desc, arglist[1:])
theargs.program = arglist[0]
theargs.version = chmutil.__version__
core.setup_logging(logger, loglevel=theargs.loglevel)
try:
return _check_chm_job(theargs)
finally:
logging.shutdown()
