parser.add_argument('--max',action='append',default=[],help='Take the maximum of the given column. Include switch multiple times for multiple columns')
parser.add_argument('--median',action='append',default=[],help='Take the median of the given column. Include switch multiple times for multiple columns')
parser.add_argument('--std',action='append',default=[],help='Take the standard deviation of the given column. Include switch multiple times for multiple columns')
parser.add_argument('--count',action='append',nargs='*',default=[],help='Number in partition. Optionally specify values to count equal to (valid value otherwise). For instance "--count multiid 0 1" would count the number of 0 (unindexed) patterns and the number of 1st indexed crystals which would total the number of frames. "--count" without a column will return the number of rows in the file/partition. Currently just working for numeric columns')
parser.add_argument('--countGE',action='append',nargs=2,default=[],help='Count number in column greater than equal to value. Two arguments: col name, value')
parser.add_argument('--partnum',default=[],nargs='+',type=int,help='The number of partitions to create. Note that empty partitions are not output so the total output may be less than this number. This defaults to ten for continuous fields or all possible for categorical fields. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partmin',default=[],nargs='+',type=float,help='Set the minimum of the range to be partitioned. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partmax',default=[],nargs='+',type=float,help='Set the maximum of the range to be partitioned. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partition',default=None,nargs='+',help='Field name(s) to partition on. Partitioning only occurs if this is provided, otherwise partnum, partmin, and partmax are not used. Partition names appended to outfile as outfile_nm')
parser.add_argument('-o','--out',type=argparse.FileType('w'),default=sys.stdout,help='the output file. If not provided, will be standard out.')
parser.add_argument('infile',help='the dat file (or a .npz file output from datview/datexport)')
args=parser.parse_args()
model=DataModel(args.infile,args.group,cfg=ModelConfig(args.cfg))
if args.filter is not None:
model.loadFilters(args.filter)
args.model=model
if args.cols is None:
args.cols=[]
for c in args.average:
args.cols.append("average_"+c)
for c in args.averageN:
args.cols.append("averageN_"+c)
for c in args.min:
args.cols.append("min_"+c)
for c in args.minN:
args.cols.append("minN_"+c)
parser.add_argument('--partition',default=None,nargs='+',help='Field name(s) to partition on. Partitioning only occurs if this is provided, otherwise partnum, partmin, and partmax are not used. Partition names appended to outfile as outfile_nm')
parser.add_argument('--histclipmax',default=None,type=float,help='Values above are set to 0')
parser.add_argument('--histclipmin',default=None,type=float,help='Values below are set to 0')
parser.add_argument('--flatten',action='store_true',help='Sum the array to a single value after clipping')
parser.add_argument('--savelst',action='store_true',help='Assuming flat data, save output as text file columns')
parser.add_argument('--background',default=None,help='Subtract before histogram clipping, should be h5 file with same fields.')
parser.add_argument('infile',help='the dat file (or a .npz file output from datview/datexport)')
parser.add_argument('outfile',help='the output file, use %%s to show location of partition name. Will be h5 format')
args=parser.parse_args()
model=DataModel(args.infile,args.group,cfg=ModelConfig(args.cfg))
if args.filter is not None:
model.loadFilters(args.filter)
assert model.canSaveLst()
args.model=model
partitions=None
if args.partition is not None:
partitions=model.partitionMulti(args.partition,args.partmin,args.partmax,args.partnum)
if '%s' not in args.outfile and not args.savelst:
args.outfile +="_%s"
backgrounds=None
if args.background is not None:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: Natasha Stander
import sys,os
import argparse
import numpy as np
sys.path.append(os.path.dirname(sys.path[0]))
from api.datamodel import DataModel
from api.modelcfg import ModelConfig
parser=argparse.ArgumentParser(description='Show the min (with -1), min(no -1), mean (no -1), max of each column.')
parser.add_argument('--cfg',default=None,help='Use the provided configuration file (xml) instead of the default one. Default one is found in api/modelcfg.xml')
parser.add_argument('infile',help='the dat file')
args=parser.parse_args()
model=DataModel(args.infile,None,cfg=ModelConfig(args.cfg))
print("name\tmin (all)\tmin (no -1)\tmean\tmax")
for col in model.cols:
print (col,np.min(model.data[col]), model.fieldmin(col), np.mean(model.data[col][model.data[col]!= -1]),np.max(model.data[col]),sep="\t")
parser.add_argument('--group',default=None,help='The group file output by groupgen.py (groupcfg.txt)')
parser.add_argument('--filter',default=None,help='A filter file to load. Filter files are XML format. The first Between filter in the file for a field will be updated with selection.')
parser.add_argument('--sort',default=None,nargs='+',help='One or more fields to sort the output by. Field names must match the header of the dat file. Multiple arguments accepted so don\'t use as last switch before inputs.')
parser.add_argument('--reversesort',action="store_true",help='Sort descending instead of ascending')
parser.add_argument('--cfg',default=None,help='Use the provided configuration file (xml) instead of the default one. Default one is found in api/modelcfg.xml')
parser.add_argument('--partnum',default=[],nargs='+',type=int,help='The number of partitions to create. Note that empty partitions are not output so the total output may be less than this number. This defaults to ten for continuous fields or all possible for categorical fields. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partmin',default=[],nargs='+',type=float,help='Set the minimum of the range to be partitioned. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partmax',default=[],nargs='+',type=float,help='Set the maximum of the range to be partitioned. Accepts multiple arguments, one for each field. Unspecified arguments default to None')
parser.add_argument('--partition',default=None,nargs='+',help='Field name(s) to partition on. Partitioning only occurs if this is provided, otherwise partnum, partmin, and partmax are not used. Partition names appended to outfile as outfile_nm')
parser.add_argument('datfile',help='the dat file (or a .npz file output from datview/datexport)')
args=parser.parse_args()
# Load Dat File
model=DataModel(args.datfile,args.group,cfg=ModelConfig(args.cfg))
if args.filter is not None:
model.loadFilters(args.filter)
if args.sort is not None:
model.sortlst = args.sort
model.reverseSort = args.reversesort
if args.cols is None:
if len(args.plot) == 1:
args.cols = 1
else:
args.cols=model.cfg.histperrow
if args.rows is None:
if len(args.plot) == 1:
args.rows = 1
else:
'--partition',
default=None,
nargs='+',
help=
'Field name(s) to partition on. Partitioning only occurs if this is provided, otherwise partnum, partmin, and partmax are not used. Partition names appended to outfile as outfile_nm'
)
parser.add_argument(
'infile',
help='the dat file (or a .npz file output from datview/datexport)')
parser.add_argument(
'outfile',
help='the output file, format automatically determined by line ending')
args = parser.parse_args()
model = DataModel(args.infile, args.group, cfg=ModelConfig(args.cfg))
if args.outfile.endswith(".npz"):
model.saveAllNumpy(args.outfile)
exit()
if args.filter is not None:
model.loadFilters(args.filter)
if args.sort is not None:
model.sortlst = args.sort
if args.limit is not None:
model.limit = args.limit
model.limitModeRandom = args.limitmode == "random"
model.reverseSort = args.reversesort
partitions = None
if args.partition is not None:
def __init__(self, datfile, groupfile, filterfile, cfg, geom, mask):
QMainWindow.__init__(self)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.setAttribute(Qt.WA_DeleteOnClose)
self.setWindowTitle("Datview: %s" % datfile)
config = ModelConfig(cfg)
self.model = DataModel(datfile, groupfile, cfg=config)
self.histcolumns = config.histperrow
if filterfile is not None:
self.model.loadFilters(filterfile)
self.ui.actionSave_Dat.triggered.connect(self.onSaveDat)
self.ui.actionSave_List.setEnabled(self.model.canSaveLst())
self.ui.actionSave_List.triggered.connect(self.onSaveLst)
self.ui.actionSave_Stream.setEnabled(self.model.canSaveStream())
self.ui.actionSave_Stream.triggered.connect(self.onSaveStream)
self.ui.actionSave_Numpy.setEnabled(self.model.canSaveNumpy())
self.ui.actionSave_Numpy.triggered.connect(self.onSaveNumpy)
self.ui.actionScatter.triggered.connect(self.onShowScatter)
self.ui.action2D_Histogram.triggered.connect(self.onShowHist2d)
self.ui.actionPixel.triggered.connect(self.onShowPixelPlot)
self.ui.actionAggregated_Plot.triggered.connect(self.onShowAggPlot)
self.ui.actionOpen.setVisible(False)
self.ui.actionSave_Plot.setVisible(False)
self.ui.actionComparison_Scatter.setVisible(
self.model.hasComparisons())
self.ui.actionComparison_Scatter.triggered.connect(
self.onShowCmpScatter)
self.ui.actionComparison_2D_Histogram.setVisible(
self.model.hasComparisons())
self.ui.actionComparison_2D_Histogram.triggered.connect(
self.onShowCmpHist2d)
self.filtmessage = QLabel(self)
self.ui.statusbar.addWidget(self.filtmessage)
self.onFilterChange()
self.model.filterchange.connect(self.onFilterChange)
# Histogram Menu
self.checkedhistograms = []
self.cachedHistograms = {}
self.ui.menuHistogram_Bar.removeAction(self.ui.actionReset)
self.addHistogramMenu(
self.model.cfg.defaultHistograms & set(self.model.cols), True)
self.ui.menuHistogram_Bar.addSeparator()
self.addHistogramMenu(
set(self.model.cols) - self.model.cfg.defaultHistograms -
self.model.cfg.internalCols, False)
self.placeHistograms()
# Filter Panel
self.controlPanel = MyControlPanel(self.model, parent=self)
self.controlPanel.setWindowFlags(Qt.Window)
self.ui.actionViewControls.triggered.connect(self.controlPanel.show)
self.ui.actionSave_Filters.triggered.connect(
self.controlPanel.onSaveFilters)
# Item Viewer
itemviewer = MyItemViewer(self.model, geom, mask, parent=self)
itemviewer.setWindowFlags(Qt.Window)
self.ui.actionItem_Viewer.triggered.connect(itemviewer.show)
self.controlPanel.flagselected.connect(itemviewer.model.setRow)
if qt5:
self.ui.plotScrollArea.viewport().installEventFilter(self)
class MyMainWindow(QMainWindow):
def __init__(self, datfile, groupfile, filterfile, cfg, geom, mask):
QMainWindow.__init__(self)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.setAttribute(Qt.WA_DeleteOnClose)
self.setWindowTitle("Datview: %s" % datfile)
config = ModelConfig(cfg)
self.model = DataModel(datfile, groupfile, cfg=config)
self.histcolumns = config.histperrow
if filterfile is not None:
self.model.loadFilters(filterfile)
self.ui.actionSave_Dat.triggered.connect(self.onSaveDat)
self.ui.actionSave_List.setEnabled(self.model.canSaveLst())
self.ui.actionSave_List.triggered.connect(self.onSaveLst)
self.ui.actionSave_Stream.setEnabled(self.model.canSaveStream())
self.ui.actionSave_Stream.triggered.connect(self.onSaveStream)
self.ui.actionSave_Numpy.setEnabled(self.model.canSaveNumpy())
self.ui.actionSave_Numpy.triggered.connect(self.onSaveNumpy)
self.ui.actionScatter.triggered.connect(self.onShowScatter)
self.ui.action2D_Histogram.triggered.connect(self.onShowHist2d)
self.ui.actionPixel.triggered.connect(self.onShowPixelPlot)
self.ui.actionAggregated_Plot.triggered.connect(self.onShowAggPlot)
self.ui.actionOpen.setVisible(False)
self.ui.actionSave_Plot.setVisible(False)
self.ui.actionComparison_Scatter.setVisible(
self.model.hasComparisons())
self.ui.actionComparison_Scatter.triggered.connect(
self.onShowCmpScatter)
self.ui.actionComparison_2D_Histogram.setVisible(
self.model.hasComparisons())
self.ui.actionComparison_2D_Histogram.triggered.connect(
self.onShowCmpHist2d)
self.filtmessage = QLabel(self)
self.ui.statusbar.addWidget(self.filtmessage)
self.onFilterChange()
self.model.filterchange.connect(self.onFilterChange)
# Histogram Menu
self.checkedhistograms = []
self.cachedHistograms = {}
self.ui.menuHistogram_Bar.removeAction(self.ui.actionReset)
self.addHistogramMenu(
self.model.cfg.defaultHistograms & set(self.model.cols), True)
self.ui.menuHistogram_Bar.addSeparator()
self.addHistogramMenu(
set(self.model.cols) - self.model.cfg.defaultHistograms -
self.model.cfg.internalCols, False)
self.placeHistograms()
# Filter Panel
self.controlPanel = MyControlPanel(self.model, parent=self)
self.controlPanel.setWindowFlags(Qt.Window)
self.ui.actionViewControls.triggered.connect(self.controlPanel.show)
self.ui.actionSave_Filters.triggered.connect(
self.controlPanel.onSaveFilters)
# Item Viewer
itemviewer = MyItemViewer(self.model, geom, mask, parent=self)
itemviewer.setWindowFlags(Qt.Window)
self.ui.actionItem_Viewer.triggered.connect(itemviewer.show)
self.controlPanel.flagselected.connect(itemviewer.model.setRow)
if qt5:
self.ui.plotScrollArea.viewport().installEventFilter(self)
def eventFilter(self, source, event):
if (event.type() == QEvent.Wheel
and source is self.ui.plotScrollArea.viewport()):
return True
return super(QMainWindow, self).eventFilter(source, event)
def addHistogramMenu(self, lst, checked=False):
for col in sorted(lst, key=self.model.prettyname):
act = QAction(self.model.prettyname(col), self)
act.setCheckable(True)
act.setChecked(checked)
self.ui.menuHistogram_Bar.addAction(act)
act.triggered.connect(
lambda checked, field=col: self.onHistAction(field, checked))
if checked:
self.checkedhistograms.append(col)
def placeHistograms(self):
spot = 0
for field in self.checkedhistograms:
if field in self.cachedHistograms:
h = self.cachedHistograms[field]
else:
h = MyFigure(parent=self.ui.scrollAreaWidgetContents)
h.histogram(model=self.model, field=field)
self.cachedHistograms[field] = h
if spot < self.histcolumns:
self.ui.gridLayout.setColumnStretch(spot, 1)
if spot % self.histcolumns == 0:
self.ui.gridLayout.setRowStretch(int(spot / self.histcolumns),
1)
self.ui.gridLayout.addWidget(h, int(spot / self.histcolumns),
spot % self.histcolumns)
h.setVisible(True)
spot += 1
def onHistAction(self, field, checked):
if checked:
self.checkedhistograms.append(field)
else:
self.checkedhistograms.remove(field)
self.cachedHistograms[field].setVisible(False)
self.placeHistograms()
def onFilterChange(self):
self.filtmessage.setText(
'%d of %d Selected' %
(len(self.model.filtered), len(self.model.data)))
def getSaveName(self, title, filter):
name = QFileDialog.getSaveFileName(self, title, filter=filter)
if qt5:
if name:
return name[0]
elif name is not None and len(name):
return name
return None
def onSaveDat(self):
name = self.getSaveName('Save Selected As Dat File', '*.dat')
if name:
self.model.saveByPartitions(name, self.model.saveSelDat,
self.controlPanel.partWidget.current())
def onSaveLst(self):
name = self.getSaveName('Save Selected As List File', '*.lst')
if name:
self.model.saveByPartitions(name, self.model.saveSelLst,
self.controlPanel.partWidget.current())
def onSaveStream(self):
name = self.getSaveName('Save Selected As Stream File', '*.stream')
if name:
try:
self.model.saveByPartitions(
name, self.model.saveSelStream,
self.controlPanel.partWidget.current())
except FileNotFoundError as err:
QMessageBox.warning(self, "Unable to Save", str(err))
def onSaveNumpy(self):
name = self.getSaveName('Save ALL as compressed numpy file', '*.npz')
if name:
self.model.saveAllNumpy(name)
def onShowScatter(self):
d = MyScatterDialog(self.model, self)
d.exec()
def onShowCmpScatter(self):
d = MyCompareScatterDialog(self.model, self)
d.exec()
def onShowHist2d(self):
d = MyHist2dDialog(self.model, self)
d.exec()
def onShowCmpHist2d(self):
d = MyCompare2DHistDialog(self.model, self)
d.exec()
def onShowPixelPlot(self):
d = MyPixelPlotDialog(self.model, self)
d.exec()
def onShowAggPlot(self):
d = MyAggPlotDialog(self.model, self)
d.exec()