def cleanup(configuration, directory):
'''
Removes any files that are not part of the results and were produced
in any of the previous steps.
:param configuration:
:param directory:
:return:
'''
print "Cleaning temporary files for %s" % directory
os.chdir(directory)
if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks'):
for filename in glob.glob('*.vtk'):
os.remove(filename)
if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot'):
for filename in glob.glob('*.gp'):
os.remove(filename)
if (os._exists("visitlog.py")):
os.remove("visitlog.py")
os.chdir("..")
return
def check_configuration(configuration):
'''
Checks the consistency of the given configuration and hotfixes
:param configuration:
:return:
'''
# Check the configuration
dimensions = utils.getValueForKeyPath(configuration, 'data.dimensions')
if (not dimensions):
print "ERROR:configuration must contain 'dimensions'"
return -1
if (not len(dimensions) in [2, 3]):
print "ERROR:Can only process 2D or 3D"
return -1
# Check that visualiseTracks has .vtk to work from
postprocessing = utils.getValueForKeyPath(configuration, 'postprocessing')
if postprocessing:
tracks = utils.getValueForKeyPath(postprocessing, 'tracks')
if tracks:
if utils.getValueForKeyPath(tracks, 'visualiseTracks') and not utils.getValueForKeyPath(tracks,
'meanie3D-trackstats.vtk_tracks'):
print "WARNING: tracks.visualiseTracks = True but tracks.meanie3D-trackstats.vtk_tracks = False. Correcting."
utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks', True)
# Complement vtk_dimensions to make our life a little easier down the road.
if __have_visit__:
vtkDimensions = utils.getValueForKeyPath(configuration, 'data.vtkDimensions')
if vtkDimensions:
vtkDimString = ",".join(vtkDimensions)
utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtkDimensions',
vtkDimString)
# Make sure that plotStats has gnuplot files to work with
if utils.getValueForKeyPath(tracks, 'plotStats') and not utils.getValueForKeyPath(tracks,
'meanie3D-trackstats.gnuplot'):
print "WARNING: track.plot_stats = True but tracks.meanie3D-trackstats.gnuplot = False. Correcting."
utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot', True)
def compile_and_run_template(templatePath, configuration, replacements, directory):
'''
Compiles the gievn template into an executable python
script and runs it through visit.
:param templatePath:
:param configuration:
:param replacements:
:param directory:
:return:True if the command was successful, False else.
'''
scriptFilename = tempfile.mktemp() + ".py"
# scriptFilename = os.path.abspath("generated.py")
# print "\tWriting python script for visualisation to: " + scriptFilename
with open(templatePath) as infile, open(scriptFilename, 'w') as outfile:
for line in infile:
for src, target in replacements.iteritems():
line = line.replace(src, target)
outfile.write(line)
# print "\tDone."
# Compile command line params for visualisation
params = "-s %s" % scriptFilename
runHeadless = utils.getValueForKeyPath(configuration, 'postprocessing.runVisitHeadless')
if runHeadless:
params = "-cli -nowin %s" % params
# change into directory
os.chdir(directory)
# Run visualisation
returnCode = -1
try:
print "Executing visit: " + params
returnCode = external.execute_command('visit', params, silent=True)
except:
print "ERROR:%s" % sys.exc_info()[0]
# Change back out
os.chdir('..')
return (returnCode == 0)
def run_trackstats(configuration, directory):
'''
Runs the meanie3D-trackstats command.
:param configuration:
:param directory:
:return:
'''
pconf = configuration['postprocessing']
if not 'tracks' in pconf:
print "Skipping meanie3D-trackstats because 'tracks' configuration is missing"
return False
tracks = pconf['tracks']
if not 'meanie3D-trackstats' in tracks:
print "Skipping meanie3D-trackstats because 'tracks.meanie3D-trackstats' configuration is missing"
return False
if utils.getValueForKeyPath(configuration,'skip_trackstats'):
print "Skipping meanie3D-trackstats because --skip-trackstats is present"
return False
print "Running meanie3D-trackstats for %s" % directory
conf = tracks['meanie3D-trackstats']
os.chdir(directory)
# Assemble command line params
haveFiles = True
params = []
if (conf['dictionary'] == True):
params.append("-t")
haveFiles = haveFiles and os.path.exists('track-dictionary.json')
if (conf['gnuplot'] == True):
params.append("-g")
haveFiles = haveFiles and os.path.exists('plot_stats.gp')
if (conf['length'] == True):
params.append("--create-length-statistics")
haveFiles = haveFiles and os.path.exists("lengths.txt")
haveFiles = haveFiles and os.path.exists("lengths-hist.txt")
if (conf['length_classes']):
params.append("--length-histogram-classes=%s" % conf['length_classes'])
if (conf['speed'] == True):
params.append("--create-speed-statistics")
haveFiles = haveFiles and os.path.exists("speeds.txt")
haveFiles = haveFiles and os.path.exists("speeds-hist.txt")
if (conf['speed_classes']):
params.append("--speed-histogram-classes=%s" % conf['speed_classes'])
if (conf['direction'] == True):
params.append("--create-direction-statistics")
haveFiles = haveFiles and os.path.exists("directions.txt")
haveFiles = haveFiles and os.path.exists("directions-hist.txt")
if (conf['direction_classes']):
params.append("--direction-histogram-classes=%s" % conf['direction_classes'])
if (conf['size'] == True):
params.append("--create-cluster-statistics")
haveFiles = haveFiles and os.path.exists("sizes.txt")
haveFiles = haveFiles and os.path.exists("sizes-hist.txt")
if (conf['size_classes']):
params.append("--cluster-histogram-classes=%s" % conf['size_classes'])
if (conf['cumulated'] == True):
params.append("--create-cumulated-size-statistics")
if (conf['cumulated_classes']):
params.append("--size-histogram-classes=%s" % conf['cumulated_classes'])
if (conf['vtk_tracks'] == True):
params.append("--write-center-tracks-as-vtk ")
# Can' be sure in this case, better safe than sorry
haveFiles = False
if (__have_visit__ and conf['vtkDimensions']):
params.append("--vtk-dimensions=%s" % conf['vtkDimensions'])
params.append("-s netcdf")
if configuration['resume'] and haveFiles:
print "Skipping meanie3D-trackstats"
os.chdir("..")
return True
return_code = -1
try:
print "meanie3D-trackstats %s" % (" ".join(params))
return_code = external.execute_command("meanie3D-trackstats", " ".join(params), silent=True)
except:
print "ERROR:%s" % sys.exc_info()[0]
raise
os.chdir("..")
return (return_code == 0)
def run(configuration):
'''
Runs postprocessing steps according to the section
'postprocessing' in the configuration.
:param configuration:
:return:
'''
# Check configuration section 'postprocessing'
check_configuration(configuration)
if not 'postprocessing' in configuration:
return
# In case scale parameters were given, the output dirs are scaleXYZ.
# Otherwise it's 'clustering'. To be safe, iterate over both
scales = configuration['scales']
if scales:
directories = ['scale%s' % (scale) for scale in scales]
else:
directories = ['clustering']
print "Processing directories: %s" % str(directories)
for directory in directories:
if os.path.isdir(directory):
print "Processing %s" % directory
else:
print "ERROR:%s is not a directory!" % directory
continue
# run the track statistics
if configuration['time_operations']:
print "Running trackstats ..."
start_time = time.time()
if (run_trackstats(configuration, directory)):
if configuration['time_operations']:
print "Finished. (%.2f seconds)" % (time.time()-start_time)
# Copy HTML files
copy_html_files(configuration, directory)
# run the stats plotting
if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.plotStats'):
plot_trackstats(configuration, directory)
# run the track visualisations
if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.tracks.visualiseTracks'):
if configuration['time_operations']:
print "Visualising tracks ..."
start_time = time.time()
visualise_tracks(configuration, directory)
if configuration['time_operations']:
print "Finished. (%.2f seconds)" % (time.time()-start_time)
if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.clusters.visualiseClusters'):
if configuration['time_operations']:
print "Visualising clusters ..."
start_time = time.time()
visualise_clusters(configuration, directory)
if configuration['time_operations']:
print "Finished. (%.2f seconds)" % (time.time()-start_time)
cleanup(configuration, directory)
if (utils.getValueForKeyPath(configuration, 'postprocessing.tracks.runScaleComparison')):
run_comparison(configuration)
def plot_stats_comparison(configuration):
'''
Plots comparison charts for each of the parameters to meanie3D-trackstats,
namely--length, --directions, --speed and --direction
:param configuration:
:return:
'''
scales = configuration['scales']
if not scales:
print "No scales found, no comparison is done."
return False
pconf = configuration['postprocessing']
if not pconf:
return False
if not utils.getValueForKeyPath(pconf,'tracks.runScaleComparison'):
return False
conf = utils.getValueForKeyPath(pconf, 'tracks.meanie3D-trackstats')
print "Plotting .eps files for comparison"
# create a tempfile for gnuplot
f = open('plot_stats_comparison.gp', 'w')
f.write('set terminal png\n')
f.write('set style fill transparent pattern 4 border 2\n')
# track length
if utils.getValueForKeyPath(conf,'length'):
f.write('set title "Distribution of track length"\n')
f.write('set xlabel "Track length in [#steps]"\n')
f.write('set ylabel "log(#tracks)"\n')
f.write('set xtics 5\n')
f.write('set xrange [2:50]\n')
f.write('set logscale y\n')
f.write('set output "lengths-hist-comparison.png"\n')
f.write('plot ')
for i in range(0, len(scales)):
scale = scales[i]
f.write('"scale%s/lengths-hist.txt" with linespoints title "t=%s"' % (scale,scale))
if (i < len(scales)-1):
f.write(",")
f.write('\nunset logscale y\n')
f.write('set autoscale x\n')
# cluster size
if utils.getValueForKeyPath(conf,'size'):
f.write('set title "Distribution of cluster size"\n')
f.write('set xlabel "log(cluster size in [#gridpoints])"\n')
f.write('set ylabel "number of clusters"\n')
f.write('set logscale x\n')
f.write('set xrange [10:10000]\n')
f.write('set xtics auto\n')
f.write('set output "sizes-hist-comparison.png"\n')
f.write('plot ')
for i in range(0, len(scales)):
scale = scales[i]
f.write('"scale%s/sizes-hist.txt" with linespoints title "t=%s"' % (scale,scale))
if (i < len(scales)-1):
f.write(",")
f.write('\nunset logscale x\n')
f.write('set autoscale x\n')
# speed
if utils.getValueForKeyPath(conf,'speed'):
f.write('set title "Distribution of cluster speeds"\n')
f.write('set xlabel "Cluster speed in [m/s]"\n')
f.write('set ylabel "Number of clusters"\n')
f.write('set xtics 5\n')
f.write('set output "speeds-hist-comparison.png"\n')
f.write('plot ')
for i in range(0, len(scales)):
scale = scales[i]
f.write('"scale%s/speeds-hist.txt" with linespoints title "t=%s"' % (scale,scale))
if (i < len(scales)-1):
f.write(",")
f.write("\n")
# directions
if utils.getValueForKeyPath(conf,'direction'):
f.write('set title "Distribution of tracking direction"\n')
f.write('set xlabel "Cluster direction in [deg]"\n')
f.write('set ylabel "Number of clusters"\n')
f.write('set xtics 30\n')
f.write('set xrange [15:360]\n')
f.write('set output "directions-hist-comparison.png"\n')
f.write('plot ')
for i in range(0, len(scales)):
scale = scales[i]
f.write('"scale%s/directions-hist.txt" with linespoints title "t=%s"' % (scale,scale))
if (i < len(scales)-1):
f.write(",")
f.write("\n")
f.close()
return_code = -1
try:
return_code = external.execute_command("gnuplot", "plot_stats_comparison.gp")
except:
print "ERROR:%s" % sys.exc_info()[0]
os.chdir("..")
return (return_code == 0)