def _handleStartServer(self): if not NGinxSetupOps.commandExists(): self.mainWindow.updateStatusBar( u'ERROR: No NGinx command available') print 'NO COMMAND' return if NGinxRunOps.isRunning(): self.mainWindow.updateStatusBar( u'ERROR: An NGinx process is already active') print 'ALREADY RUNNING' return path = self.rootPath self.mainWindow.appConfig.set(AppConfigEnum.ROOT_PATH, path) recentPaths = self.mainWindow.appConfig.get(AppConfigEnum.RECENT_PATHS, []) ListUtils.addIfMissing(path, recentPaths, reorder=True, frontOrdering=True) self.mainWindow.appConfig.set(AppConfigEnum.RECENT_PATHS, recentPaths) thread = NGinxRemoteThread(self, rootPath=path) self._serverThread = thread thread.start() self.mainWindow.updateStatusBar(u'NGinx server now active') self._updateDisplay()
def _cloneValue(cls, value): if isinstance(value, list): from pyaid.list.ListUtils import ListUtils return ListUtils.clone(value) elif isinstance(value, tuple): return tuple(ListUtils.clone(value)) elif isinstance(value, dict): return cls.clone(value) return value
def _cloneValue(cls, value): from pyaid.list.ListUtils import ListUtils if isinstance(value, list): return ListUtils.clone(value) elif isinstance(value, tuple): return tuple(ListUtils.clone(value)) elif isinstance(value, dict): return cls.clone(value) return value
def _executeDeployment(self, deployType): rootPath = self.mainWindow.getWidgetFromID('home').rootPath recentPaths = self.mainWindow.appConfig.get(AppConfigEnum.RECENT_PATHS, []) ListUtils.addIfMissing(rootPath, recentPaths, reorder=True, frontOrdering=True) self.mainWindow.appConfig.set(AppConfigEnum.RECENT_PATHS, recentPaths) self._closeBtn.setEnabled(False) thread = SiteDeploymentThread(self, rootPath=rootPath, deployType=deployType) self._serverThread = thread thread.execute( logCallback=self._handleLogData, callback=self._handleDeploymentExecutionComplete) thread.start() self.mainWindow.updateStatusBar(u'Deployment in progress')
def compare(cls, a, b): if a == b: return True if a is None or b is None: return False if len(a.keys()) != len(b.keys()): return False for name, value in a.iteritems(): if name not in b: return False # Compare dict values if isinstance(value, dict): if isinstance(b[name], dict): if not cls.compare(value, b[name]): return False # Compare list and tuples if isinstance(value, list) or isinstance(value, tuple): from pyaid.list.ListUtils import ListUtils if not ListUtils.compare(value, b[name]): return False if value != b[name]: return False return True
def compare(cls, a, b, onlyCommonKeys =False): if a == b: return True if a is None or b is None: return onlyCommonKeys if not onlyCommonKeys and len(a.keys()) != len(b.keys()): return False for name, value in cls.iter(a): if name not in b: if onlyCommonKeys: continue return False # Compare dict values if isinstance(value, dict): if isinstance(b[name], dict): if not cls.compare(value, b[name], onlyCommonKeys=onlyCommonKeys): return False # Compare list and tuples if isinstance(value, list) or isinstance(value, tuple): from pyaid.list.ListUtils import ListUtils if not ListUtils.compare(value, b[name]): return False if value != b[name]: return False return True
def _postAnalyze(self): """_postAnalyze doc...""" ratios = [] for name, curve in DictUtils.iter(self.data): segments = curve.segments for i in ListUtils.rangeOn(segments): segment = segments[i] segmentLine = segment.line # If this is an extrapolated segment, use the length from the neighboring segment # instead of the artificial length of this segment. if segment == segments[0]: segmentLine = segments[i + 1].line elif segment == segments[-1]: segmentLine = segments[i - 1].line for pairData in segment.pairs: projectionLine = pairData["line"] ratios.append(100.0 * projectionLine.length.raw / segmentLine.length.raw) h = Histogram( data=ratios, binCount=50, xLabel="Projection/Stride Ratio (%)", title="Relative Stride to Projection Length Ratios", ) h.shaveDataToXLimits() self._paths.append(h.save(path=self.getTempFilePath(extension="pdf"))) self.mergePdfs(self._paths, "Curve-Projection.pdf")
def _process(self): """_process doc...""" length = 0.0 for segment in self.segments: length = max(length, segment.offset) # Sort the paired segments by distance from the segment start position to order them # properly from first to last if segment.pairs: ListUtils.sortDictionaryList(segment.pairs, 'distance', inPlace=True) for p in segment.pairs: self._resolveSpatialCoincidences(p) length = max(length, segment.offset + segment.pairs[-1]['distance']) self.length = length
def __call__(self, *args, **kwargs): try: return self._createApp( ArgsUtils.getAsDict('environ', kwargs, args, 0), ArgsUtils.get('start_response', None, kwargs, args, 1) ) except Exception as err: self.logger.writeError([ 'ERROR: Application Creation Failed', 'ARGS: %s' % ListUtils.prettyPrint(args), 'KWARGS: %s' % DictUtils.prettyPrint(kwargs) ], err) raise
def loadWidgetFile(cls, widget, names =None, loadAsWidget =True, target =None): """ Loads the UI file for the widget from its resource path. """ widgetName = widget.__class__.__name__ lookups = ListUtils.asList(names, allowTuples=False) + [widgetName, 'widget', 'main', 'gui'] if not target: target = widget try: path = widget.getResourcePath(isDir=True, inApp=True) except Exception: raise IOError('[ERROR]: No widget resource path found for "%s" widget' % (widgetName)) if not os.path.exists(path): try: path = widget.getResourcePath(isDir=True) except Exception: raise IOError( '[ERROR]: No widget resource path found for "%s" widget' % (widgetName)) if not os.path.exists(path): raise IOError( '[ERROR]: Missing widget resource path [%s]: %s' % (widgetName, path)) result = None for item in lookups: result = cls.loadUiFile( target=target, pathNoExtension=FileUtils.makeFilePath(path, item, isFile=True), loadAsWidget=loadAsWidget) if result: break if not result: raise Exception('[ERROR]: No UI file found at: ' + path) if result is not target: layout = QtGui.QVBoxLayout() layout.addWidget(result) target.setLayout(layout) elements = [] for item in dir(result): item = getattr(result, item) if isinstance(item, QtGui.QWidget): elements.append(item) else: layout = target.layout() elements = None return {'widget':result, 'layout':layout, 'elements':elements}
def addChildAt(self, tag, index =0): index = ListUtils.getAbsoluteIndex(index, self._children) try: currentIndex = self._children.index(tag) if currentIndex < index: self._children.remove(tag) self._children.insert(index, tag) elif currentIndex > index: self._children.insert(index, tag) self._children.remove(tag) tag.parent = self return True except Exception, err: pass
def clone(cls, item): out = dict() for key, value in cls.iter(item): if isinstance(value, dict): # Clone dict recursively out[key] = cls.clone(value) elif isinstance(value, list) or isinstance(value, tuple): # Clone lists and tuples from pyaid.list.ListUtils import ListUtils out[key] = ListUtils.clone(value) else: out[key] = value return out
def _postAnalyze(self): self.logger.write('%s gauge calculated tracks' % self._count) self._trackwayCsv.save() csv = CsvWriter( path=self.getPath('Simple-Gauge-Errors.csv', isFile=True), autoIndexFieldName='Index', fields=[ ('uid', 'UID'), ('fingerprint', 'Fingerprint') ]) for track in self._errorTracks: csv.createRow(uid=track.uid, fingerprint=track.fingerprint) csv.save() if self._errorTracks: self.logger.write('Failed to calculate gauge for %s tracks' % len(self._errorTracks)) csv = CsvWriter( path=self.getPath('Simple-Gauge-Ignores.csv', isFile=True), autoIndexFieldName='Index', fields=[ ('uid', 'UID'), ('fingerprint', 'Fingerprint') ]) for track in self._ignoreTracks: csv.createRow(uid=track.uid, fingerprint=track.fingerprint) csv.save() if self._ignoreTracks: self.logger.write('%s tracks lacked suitability for gauge calculation' % len( self._ignoreTracks)) plotData = [ ('stride', 'red', 'AU', 'Stride-Normalized Weighted'), ('pace', 'green', 'AU', 'Pace-Normalized Weighted'), ('width', 'blue', 'AU', 'Width-Normalized Weighted'), ('abs', 'purple', 'm', 'Absolute Unweighted') ] for data in plotData: out = [] source = ListUtils.sortListByIndex( source=getattr(self._trackwayGauges, StringUtils.toStr2(data[0])), index=0, inPlace=True) for item in source: out.append(PositionValue2D(x=len(out), y=item[1].value, yUnc=item[1].uncertainty)) self._plotTrackwayGauges(out, *data[1:]) self.mergePdfs(self._paths, 'Gauges.pdf')
def _processSparsenessResults(self, key): """_processSparsenessResults doc...""" index = 0 means = [] totals = [] twValues = [] tsValues = [] lows = dict(x=[], y=[], error=[], color='#666666') mids = dict(x=[], y=[], error=[], color='#33CC33') highs = dict(x=[], y=[], error=[], color='#CC3333') for uid, entry in DictUtils.iter(self.data): # For each test list in track ratings process the data and filter it into the correct # segments for plotting. data = (entry['pes'] + entry['manus']) if not key else entry[key] data = ListUtils.sortObjectList(data, 'value') index += 1 if len(data) < 2: continue average = NumericUtils.weightedAverage(*data[1:]) means.append(average) totals.extend(data[1:]) twValues.append(average.value) maxVal = data[0] for v in data[1:]: if v.value > maxVal.value: maxVal = v if maxVal.value < 15.0: target = lows elif maxVal.value < 50.0: target = mids else: target = highs for v in data[1:]: tsValues.append(v.value) target['x'].append(index) target['y'].append(v.value) target['error'].append(v.uncertainty) return lows, mids, highs, tsValues, twValues, totals
def _handleStartServer(self): if not NGinxSetupOps.commandExists(): self.mainWindow.updateStatusBar(u'ERROR: No NGinx command available') print 'NO COMMAND' return if NGinxRunOps.isRunning(): self.mainWindow.updateStatusBar(u'ERROR: An NGinx process is already active') print 'ALREADY RUNNING' return path = self.rootPath self.mainWindow.appConfig.set(AppConfigEnum.ROOT_PATH, path) recentPaths = self.mainWindow.appConfig.get( AppConfigEnum.RECENT_PATHS, [] ) ListUtils.addIfMissing(path, recentPaths, reorder=True, frontOrdering=True) self.mainWindow.appConfig.set(AppConfigEnum.RECENT_PATHS, recentPaths) thread = NGinxRemoteThread(self, rootPath=path) self._serverThread = thread thread.start() self.mainWindow.updateStatusBar(u'NGinx server now active') self._updateDisplay()
def executeCommand(cls, cmd, remote=False, shell=True, wait=False, background=False, resultObj=False): if shell and not StringUtils.isStringType(cmd): from pyaid.list.ListUtils import ListUtils cmd = ' '.join(ListUtils.itemsToString(cmd)) # Background nohup processes shouldn't PIPE and once run should immediately return if background: subprocess.Popen(cmd, shell=shell) return {'error': '', 'out': '', 'code': 0, 'command': cmd} if remote: pipe = subprocess.Popen(cmd, shell=shell, stdout=None, stderr=None, stdin=None, close_fds=False) if wait: pipe.wait() return {'error': '', 'out': '', 'code': 0, 'command': cmd} pipe = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if wait: pipe.wait() out, error = pipe.communicate() if resultObj: return cls.CMD_RESULT_NT(error=StringUtils.toUnicode(error), output=StringUtils.toUnicode(out), code=pipe.returncode, success=pipe.returncode == 0, command=cmd) return { 'error': StringUtils.toUnicode(error), 'out': StringUtils.toUnicode(out), 'code': pipe.returncode, 'command': cmd }
def cleanBytesToText(cls, source, inPlace =True): """cleanBytesToText doc...""" out = source if inPlace else dict() from pyaid.list.ListUtils import ListUtils for n, v in cls.iter(source): if isinstance(v, (tuple, list)): v = ListUtils.cleanBytesToText(v, inPlace=inPlace) elif isinstance(v, dict): v = cls.cleanBytesToText(v) else: v = StringUtils.strToUnicode(v, force=False) out[StringUtils.toStrStr(n)] = v return out
def executeCommand( cls, cmd, remote =False, shell =True, wait =False, background =False, resultObj =False ): if shell and not StringUtils.isStringType(cmd): from pyaid.list.ListUtils import ListUtils cmd = ' '.join(ListUtils.itemsToString(cmd)) # Background nohup processes shouldn't PIPE and once run should immediately return if background: subprocess.Popen(cmd, shell=shell) return {'error':'', 'out':'', 'code':0, 'command':cmd} if remote: pipe = subprocess.Popen( cmd, shell=shell, stdout=None, stderr=None, stdin=None, close_fds=False) if wait: pipe.wait() return {'error':'', 'out':'', 'code':0, 'command':cmd} pipe = subprocess.Popen( cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if wait: pipe.wait() out, error = pipe.communicate() if resultObj: return cls.CMD_RESULT_NT( error=StringUtils.toUnicode(error), output=StringUtils.toUnicode(out), code=pipe.returncode, success=pipe.returncode == 0, command=cmd) return { 'error':StringUtils.toUnicode(error), 'out':StringUtils.toUnicode(out), 'code':pipe.returncode, 'command':cmd}
def getFilteredByFlags(self, flags, files=True, directories=True): if isinstance(flags, basestring): flags = [flags] sources = [] if directories: sources += self._directories.values() if files: sources += self._files.values() sources.sort(key=lambda x: x["index"]) out = [] for src in sources: if flags is None or ListUtils.contains(src["flags"], flags): out.insert(0, src["item"]) return out
def getFilteredByFlags(self, flags, files=True, directories=True): if StringUtils.isStringType(flags): flags = [flags] sources = [] if directories: sources += self._directories.values() if files: sources += self._files.values() sources.sort(key=lambda x: x['index']) out = [] for src in sources: if flags is None or ListUtils.contains(src['flags'], flags): out.insert(0, src['item']) return out
def getFilteredByFlags(self, flags, files =True, directories =True): if StringUtils.isStringType(flags): flags = [flags] sources = [] if directories: sources += self._directories.values() if files: sources += self._files.values() sources.sort(key=lambda x:x['index']) out = [] for src in sources: if flags is None or ListUtils.contains(src['flags'], flags): out.insert(0, src['item']) return out
def _calculateSparseness(cls, spacings, reference): """ Calculates the relative sparseness from the series spacings list and the reference spacing. """ out = [] for data in spacings: # For each entry in the tests, normalize that value to the most complete (highest # track count) series to create a relative sparseness rating diff = data.value - reference.value absDiff = abs(diff) dVal = reference.value sign = 0.0 if absDiff == 0.0 else diff/absDiff unc = abs(data.uncertainty/dVal) + abs(dVal*sign - absDiff)/(dVal*dVal) out.append(NumericUtils.toValueUncertainty( value=100.0*absDiff/dVal, uncertainty=100.0*unc)) return ListUtils.sortObjectList(out, 'value')
def _processCurveDeviations(self, key, label): """_processCurveDeviations doc...""" d = [data['deviations'][key] for k, data in DictUtils.iter(self.trackwaysData)] dCurved = [] dStraight = [] for item in d: if item < 1.0: dStraight.append(item) else: dCurved.append(min(10.0, item)) plot = Histogram( data=dCurved, title='%s Trackway Deviations' % label, xLabel='Trackway Index') self._paths.insert(0, plot.save(self.getTempFilePath(extension='pdf'))) d = [(index, d[index]) for index in ListUtils.rangeOn(d)] dCurved = [] dStraight = [] for item in d: if item[-1] < 1.0: dStraight.append(item) else: dCurved.append((item[0], min(10.0, item[1])) ) self.logger.write('%s Trackways: %s/%s (%s%%)' % ( label, len(dCurved), len(dCurved) + len(dStraight), 100.0*len(dCurved)/(len(dCurved) + len(dStraight)) )) plot = MultiScatterPlot( {'data':dStraight, 'color':'blue'}, {'data':dCurved, 'color':'red'}, title='%s Trackway Deviations' % label, xLabel='Trackway Index', yLabel='Fractional Deviation') self._paths.insert(1, plot.save(self.getTempFilePath(extension='pdf')))
def _plotTrackwayGauges(self, points, color, unit, heading): histData = [] for p in points: histData.append(p.yValue.value) plot = Histogram( data=histData, title='%s Trackway Gauges' % heading, xLabel='Averaged Trackway Gauge (%s)' % unit, yLabel='Frequency', color=color) self._paths.insert(0, plot.save(self.getTempFilePath(extension='pdf'))) plot = ScatterPlot( data=ListUtils.sortObjectList(points, 'y', inPlace=True), title='%s Trackway Gauges' % heading, xLabel='Trackway Pes Count (#)', yLabel='Averaged Trackway Gauge (%s)' % unit, color=color) self._paths.insert(0, plot.save(self.getTempFilePath(extension='pdf')))
def getWeightedMeanAndDeviation(cls, *values): """ Returns the mean and standard deviation of a weighted set of values. For further info see: http://stats.stackexchange.com/questions/6534/ how-do-i-calculate-a-weighted-standard-deviation-in-excel """ if np is None: raise ImportError('NumericUtils.getMeanAndDeviation() requires Numpy') if not values: return cls.toValueUncertainty(0.0, 0.0) if isinstance(values[0], (list, tuple)): values = values[0] if not values: return cls.toValueUncertainty(0.0, 0.0) if len(values) == 1: return values[0].clone() wxs = 0.0 ws = 0.0 weights = [] for v in values: w = 1.0/(v.uncertainty*v.uncertainty) weights.append(w) wxs += w*v.value ws += w ave = wxs/ws dev = 0.0 N = len(values) for i in ListUtils.range(N): dev += weights[i]*(values[i].value - ave)**2 denom = ws*(N - 1.0)/N dev = math.sqrt(dev/denom) return cls.toValueUncertainty(value=ave, uncertainty=dev)
def prettyPrint(cls, source, delimiter = ' | ', separator = ': '): if not source: return '[EMPTY]' from pyaid.list.ListUtils import ListUtils out = [] for n,v in cls.iter(source): n = StringUtils.toUnicode(n) if isinstance(v, dict): v = '{ ' + cls.prettyPrint(v, delimiter=delimiter, separator=separator) + ' }' elif isinstance(v, StringUtils.BINARY_TYPE): v = StringUtils.strToUnicode(v) elif isinstance(v, (list, tuple)): v = ListUtils.prettyPrint(v) else: v = StringUtils.toUnicode(v) out.append(n + separator + v) out.sort(key=StringUtils.TEXT_TYPE.lower) return delimiter.join(out)
def _calculateAverageSpacing(cls, series): """ Determines the average spacing of the tracks in the track series for use as a comparative measure of sparseness to the other track series in the trackway. If the series is not ready or does not have a sufficient number of tracks, this method will return None. :param: series | TrackSeries The series on which to determine the average spacing. :return: ValueUncertainty A value uncertainty instance that represents the average spacing of the series, or None if it's the calculation is aborted. """ if not series.isReady: # Skip trackways with invalid series return None tracks = series.tracks if not tracks or len(tracks) < 2: # Ignore series with less than two tracks return None length = 0.0 uncs = [] for i in ListUtils.range(len(tracks) - 1): line = LineSegment2D( start=tracks[i].positionValue, end=tracks[i + 1].positionValue) spacing = line.length length += spacing.value uncs.append(spacing.uncertainty) unc = NumericUtils.sqrtSumOfSquares(*uncs) return NumericUtils.toValueUncertainty( value=length/float(len(tracks)), uncertainty=unc/float(len(tracks)) )
def _drawCurveSeries(cls, drawing, series): """_drawCurveSeries doc...""" line = None for i in ListUtils.rangeOn(series.tracks): track = series.tracks[i] try: nextTrack = series.tracks[i + 1] except Exception: cls._drawLine( drawing=drawing, line=line, color='#009900', opacity=0.1, startCap=False, connect=False) return line = LineSegment2D(track.positionValue, nextTrack.positionValue) cls._drawLine( drawing=drawing, line=line, color='#009900', opacity=0.1, endCap=False )
def _analyzeSeriesPair(self, sitemap, series, pairSeries): """_analyzeSeriesPair doc...""" for index in ListUtils.range(series.count): track = series.tracks[index] data = track.snapshotData aTrack = track.getAnalysisPair(self.analysisSession) aTrack.paceLength = 0.0 aTrack.paceLengthUnc = 0.0 if self.hasPace(track): measured = NumericUtils.toValueUncertainty( value=data.get(SnapshotDataEnum.PACE), uncertainty=self.MEASURED_UNCERTAINTY) else: self.noData +=1 measured = None pairTrack = self._getPairedTrack(track, series, pairSeries) if pairTrack is None: if measured is None: continue self._logUnresolvableTrack( track=track, sitemap=sitemap, message='Unable to determine pairSeries track') continue position = track.positionValue pairPosition = pairTrack.positionValue paceLine = LineSegment2D(position, pairPosition) if not paceLine.isValid: self._logUnresolvableTrack( track=track, sitemap=sitemap, message='Invalid track separation of 0.0. Ignoring track') continue entered = paceLine.length entry = dict( track=track, pairTrack=pairTrack, drawFunc=functools.partial( self._drawPaceLine, sitemap, paceLine), # Calculated distance from AI-based data entry entered=entered) if measured: # Measured distance from the catalog entry['measured'] = measured # Absolute difference between calculated and measured distance delta = entered.raw - measured.raw entry['delta'] = delta # Fractional error between calculated and measured distance entry['fractional'] = delta/measured.raw # Sigma trackDeviations between entry['deviation'] = abs(delta/math.sqrt( measured.rawUncertainty**2 + entered.rawUncertainty**2)) self.count += 1 aTrack = track.getAnalysisPair(self.analysisSession) aTrack.paceLength = entered.raw aTrack.paceLengthUnc = entered.rawUncertainty self.entries.append(entry) # self._drawPaceLine(sitemap, paceLine, ) track.cache.set('paceData', entry)
def _generateSegments(self): """_generateSegments doc...""" segments = self.segments tracks = self.series.tracks offset = 0.0 for i in ListUtils.range(len(tracks) - 1): # Create a segment For each track in the reference series line = LineSegment2D(start=tracks[i].positionValue, end=tracks[i + 1].positionValue) if not line.isValid: self._errors.append([ '[ERROR]: Invalid curve series line segment', 'START: %s at %s | %s' % ( tracks[i].fingerprint, tracks[i].positionValue.echo(True), tracks[i].uid), 'END: %s at %s | %s' % ( tracks[i + 1].fingerprint, tracks[i + 1].positionValue.echo(True), tracks[i + 1].uid) ]) continue if self.saveToAnalysisTracks: # Populate analysis track data for curve series tracks analysisTrack = tracks[i].getAnalysisPair( self.stage.analysisSession, createIfMissing=True) analysisTrack.curveSegment = len(segments) analysisTrack.segmentPosition = 0.0 analysisTrack.curvePosition = offset s = CurveProjectionSegment( index=i + 1, track=tracks[i], line=line, offset=offset) segments.append(s) offset += line.length.raw # Add segments to the beginning and end to handle overflow conditions where the paired # track series extend beyond the bounds of the reference series srcLine = segments[0].line segLine = srcLine.createPreviousLineSegment(self.EXTENSION_LENGTH) s = CurveProjectionSegment( index=0, track=None, line=segLine, offset=-self.EXTENSION_LENGTH) segments.insert(0, s) track = tracks[-1] if self.saveToAnalysisTracks: analysisTrack = track.getAnalysisPair(self.stage.analysisSession, createIfMissing=True) analysisTrack.curveSegment = len(tracks) - 1 analysisTrack.segmentPosition = 0.0 analysisTrack.curvePosition = offset srcLine = segments[-1].line segLine = srcLine.createNextLineSegment(self.EXTENSION_LENGTH) s = CurveProjectionSegment( index=-1, track=track, line=segLine, offset=offset) segments.append(s)