def __init__ (self, tss, centralNodeId, centralNode, normalised = False, parent = None, visibleOnly = True, *args): resource_usage = ConciseMonitor() QDialog.__init__ (self, *args) self.tss = tss self.centralNodeId = centralNodeId self.centralNode = centralNode self.normalised = normalised self.parent = parent self.visibleOnly = visibleOnly self.setAttribute (Qt.WA_DeleteOnClose, True) self.setWindowTitle ('Coefficient List') self.setMinimumSize (QSize (700, 400)) _layout = QVBoxLayout (self) self.view = QTableView (self) _layout.addWidget (self.view) self.model = ProfileModel (tss, centralNodeId, centralNode, normalised, parent, visibleOnly) self.view.setModel (self.model) # hide grid #self.view.setShowGrid (False) # sorting self.proxyModel = QSortFilterProxyModel () self.proxyModel.setSourceModel (self.model) self.proxyModel.setDynamicSortFilter (True) self.view.setModel (self.proxyModel) self.view.setSortingEnabled (False) self.view.horizontalHeader ().setSortIndicatorShown (False) self.view.horizontalHeader ().setClickable (False) self.view.setColumnHidden (0, True) self.view.setColumnHidden (1, True) # set column width to fit contents self.view.resizeColumnsToContents () # Add a status line self.statusLine = QLabel (self.reportStatistics ()) _layout.addWidget (self.statusLine) # Add buttons _buttonLayout = QHBoxLayout () _layout.addLayout (_buttonLayout) self.allButton = QPushButton ('All nodes') _buttonLayout.addWidget (self.allButton) QObject.connect (self.allButton, SIGNAL ('clicked ()'), self.allNodes) self.visibleButton = QPushButton ('Visible nodes only') _buttonLayout.addWidget (self.visibleButton) QObject.connect (self.visibleButton, SIGNAL ('clicked ()'), self.visibleNodes) self.okButton = QPushButton ('OK') _buttonLayout.addWidget (self.okButton) QObject.connect (self.okButton, SIGNAL ('clicked ()'), self.ok) resource_usage.report('CoefficientsProfile init') self.view.showRow (0) self.sortIt ()
def time_minimum(n): loopuse = ConciseMonitor() for i in range(n): for j in range(n): k = i if i < j: k = j loopuse = loopuse.report('loopuse - loop of i<j for minimum: %s x %s = %s iterations' % (n,n,n*n)) for i in range(n): for j in range(n): k = min(i,j) loopuse = loopuse.report('loopuse - loop of min() for minimum: %s x %s = %s iterations' % (n,n,n*n))
def tunit(verbose): ConciseMonitor.enable(False) app = Application(None,"TestTimeSeries",verbose,True) db = Database(app.dbFilePath) if verbose: print >>sys.stderr,'TimeSeriesSet.uniqueid for %s is %s' % (db.basename(),TimeSeriesSet.uniqueid(db)) tssids = TimeSeriesSet.seriessetids(db) if not len(tssids): raise Exception, "there are no TimeSeriesSets in %s" % db.filepath() tss = TimeSeriesSet(db,tssids[0]) tss.lastreldelta_quintiles() evs = tss.events() print "TimeSeries and TimeSeriesSet okay" print "EventSeries and EventSeriesSet okay"
def __init__(self, links, scaling=1): resource_usage = ConciseMonitor() self._scaling = scaling self._size = len(links) self._links = links # initialise the igraph.Graph _ig = igraph.Graph(self._size) _weights = [] for i in range(self._size): for j in range(i): _ig.add_edges((i, j)) _corr = self.getLink(i, j) _w = int(1000 * _corr) _weights.append(_w) resource_usage.report(' FruchtermanReingold: weights') x = [] for i in range(self._size): _alpha = i * math.pi * 2 / self._size x.append(10 * [math.cos(_alpha), 10 * math.sin(_alpha)]) resource_usage.report(' FruchtermanReingold: initial positions') _coordinates = self.recentre( _ig.layout_fruchterman_reingold( seed=x, weights=_weights, repulserad=10)) resource_usage.report(' FruchtermanReingold: recentre') _maxValue = 0.0 """ for i in range (len (_coordinates)): if abs (_coordinates [i] [0]) > _maxValue: _maxValue = abs (_coordinates [i] [0]) if abs (_coordinates [i] [1]) > _maxValue: _maxValue = abs (_coordinates [i] [0]) """ self._nodes = [] _maxValue = 0 if _maxValue == 0: for i in range(len(_coordinates)): _p = QPointF(_coordinates[i][0] * scaling, _coordinates[i][1] * scaling) self._nodes.append(_p) else: for i in range(len(_coordinates)): _p = QPointF(_coordinates[i][0] * scaling / _maxValue, _coordinates[i][1] * scaling / _maxValue) self._nodes.append(_p) resource_usage.report( ' FruchtermanReingold on %d nodes complete' % (self._size))
def main(): clargs = dict(debug=False, norm=True, real=True, dbfile=None, n=None, coeff=False, intensities=False, RFORMAT=False, all=False, listindex=False, seriessetid=None, orphan=False, brief=False); ai = 1 while ai < len(sys.argv): arg = sys.argv[ai] ai += 1 if arg == "-?" or arg == "--help": Showxy.help() sys.exit(1) elif arg == "-A" or arg == "--all": clargs['all'] = True clargs['n'] = 1000000 elif arg == "-C" or arg == "--coeff": clargs['coeff'] = True elif arg == "-D" or arg == "--debug": clargs['debug'] = True elif arg == "-E" or arg == "--export": clargs['export'] = True elif arg == "-i" or arg == "--index": clargs['listindex'] = True elif arg == "-I" or arg == "--intensities": clargs['intensities'] = True elif arg == "-t" or arg == "--tssid": if ai >= len(sys.argv): raise Exception("%s missing n: try --index to see an index of the database and a list of the tssids" % arg) clargs['seriessetid'] = int(sys.argv[ai]) ai += 1 elif arg == "-M" or arg == "--monitor": ConciseMonitor.enable(True) elif arg == "--n" or arg == "--nonorm": clargs['norm'] = False elif arg == "-o" or arg == "--brieforphan": clargs['orphan'] = True clargs['brief'] = True elif arg == "-O" or arg == "--orphan": clargs['orphan'] = True clargs['brief'] = False elif arg == "--RFORMAT": clargs['RFORMAT'] = True elif arg == "--r" or arg == "--noreal": clargs['real'] = False elif clargs['dbfile'] == None: clargs['dbfile'] = arg elif clargs['n'] == None: clargs['n'] = int(arg) else: raise Exception, "unrecognised arg: %s" % arg showxy = Showxy(**clargs)
def __init__(self, matrixname, coefficients): init__usage = ConciseMonitor() db = coefficients.db() tss = coefficients.tss() super(CoefficientMatrix,self).__init__(len(tss)) self._matrixname = matrixname # matrixname is one of the names in matrix_names # if matrixname != imported_matrix_name, the matrix is called a derived matrix # and as such will be created on demand by this constructor if it is not # found in the database try: db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM %s WHERE coeffid=?" % matrix_tablenames[matrixname],[coefficients.coeffid()]) except: assert matrixname != imported_matrix_name self._create_all_derived_tables(db) db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM %s WHERE coeffid=?" % derived_matrix_tablenames[matrixname], [coefficients.coeffid()]) cardinality = 0 maxabscoeff = 0 s2o = tss.seriesid2ordinal() if matrixname == 'value': issymm = coefficients.values_issymmetrical() # the original imported value matrix may or may not be symmetrical else: issymm = True # all derived matrices are symmetrical for (id1, id2, coeff) in db.cursor().fetchall(): if id1 not in s2o: raise Exception, "seriesid1=%s not found for coeffid=%s in database %s for matrix %s" % (id1, coefficients.coeffid(), db.basename(), self._matrixname) if id2 not in s2o: raise Exception, "seriesid2=%s not found for coeffid=%s in database %s for matrix %s" % (id2, coefficients.coeffid(), db.basename(), self._matrixname) if coeff: cardinality += 1 self [s2o[id1]] [s2o[id2]] = coeff if issymm: assert s2o[id1] > s2o[id2] cardinality += 1 self [s2o[id2]] [s2o[id1]] = coeff if abs(coeff) > maxabscoeff: maxabscoeff = abs(coeff) self._cardinality = cardinality self._fill = cardinality / float(self.N*self.N) self._issymmetrical = issymm self._maxabscoeff = maxabscoeff init__usage.report('%s %s init %s non-zero coeffs %.2f%% mac=%s' % ('CoefficientMatrix', self.matrixname(), self.cardinality(), self.fill(), self.mac()))
def __init__ (self, tss, thisNode, normalised = False, parent = None, visibleOnly = True, *args): resource_usage = ConciseMonitor() QDialog.__init__ (self, *args) self.tss = tss self.thisNode = thisNode self.normalised = normalised self.parent = parent self.visibleOnly = visibleOnly self.setAttribute (Qt.WA_DeleteOnClose, True) self.setWindowTitle ('Coefficients') self.setMinimumSize (QSize (700, 400)) _layout = QVBoxLayout (self) _currentRow = thisNode.index self.view = QTableView (self) _layout.addWidget (self.view) self.model = ProfileModel (tss, thisNode, normalised, parent, visibleOnly) self.view.setModel (self.model) # hide grid #self.view.setShowGrid (False) # set column width to fit contents #self.view.resizeColumnsToContents () # Colour the current row self.view.selectRow (_currentRow) self.view.showRow (_currentRow) # Add a status line for Rick self.statusLine = QLabel (self.reportStatistics ()) _layout.addWidget (self.statusLine) # Add buttons _buttonLayout = QHBoxLayout () _layout.addLayout (_buttonLayout) self.allButton = QPushButton ('All nodes') _buttonLayout.addWidget (self.allButton) QObject.connect (self.allButton, SIGNAL ('clicked ()'), self.allNodes) self.visibleButton = QPushButton ('Visible nodes only') _buttonLayout.addWidget (self.visibleButton) QObject.connect (self.visibleButton, SIGNAL ('clicked ()'), self.visibleNodes) self.okButton = QPushButton ('OK') _buttonLayout.addWidget (self.okButton) QObject.connect (self.okButton, SIGNAL ('clicked ()'), self.ok) resource_usage.report('CoefficientsProfile init')
def __init__(self, db, tss, coeffid, sparse=False): resource_usage = ConciseMonitor() self.sparse = sparse self._N = len(tss) self._db = db self._tss = tss self._coeffid = coeffid self._dict = Dict(db,"CoefficientMatrix",coeffid) # legacy: CoefficientMatrix now refers to Coefficients in the dict for this class self._matrices = dict() # all matrices: key is coefficient matrix name # public attributes: each of the named CoefficientMatrixs contained in this Coefficients object self.value = self._load('value') # original imported coefficients self.coeffs = self.value # legacy self.best = self._load('best') # best coefficients self.secondary = self._load('secondary') # secondary coefficients self.direction = self._load('direction') # direction coefficients self.norm = self._load('norm') # normalised coefficients resource_usage.report('%s #%s init %s matrices, values.cardinality=%s .fill=%.2f%% .mac=%s' % ('Coefficients', coeffid, len(matrix_names), self.cardinality(), 100*self.fill(), self.maxabscoeff()))
def task_preamble(self): if not self.enterThisFunction(): return self.resource_usage = ConciseMonitor() self.root.galaxy.view.links = [] self.root.galaxy.view.visibleNodeDetails.setNodeList([self.root.centerNode()]) try: self.root.nodes[self.root.centerNode()].hiding = False except: self.root.setCenterNode(0) self.root.nodes[0].hiding = False self.nextStep()
def nextStep(self): self.resource_usage.report("nextStep: %s" % (self.phase.__name__)) self.resource_usage = ConciseMonitor() # Report how long the last phase took ##print self.phase.__name__ if self.phase.__name__ == "task_preamble": self.START = time.time() self.BIG_START = self.START self.START = self.root.INSTRUMENT_CENTRIFUGE(self.phase.__name__, self.START) ##print self.START self.iteration = 0 # prepare for next phase self.phase = self.phases[self.phases.index(self.phase) + 1] # Tell the GUI thread to move on self.root.galaxy.view.display.emit()
def utility(): ConciseMonitor.enable(False) brief = False debug = False update = False default_dbfile = Application(None,"CoefficientMatrix Utility",False,True).dbFilePath dbfiles = [ ] for arg in sys.argv[1:]: if arg == '--help' or arg == '-?': print '%s: display coefficient matrix metrics in one or more BC database files, optionally installing post-import coefficient matrices as well' % sys.argv[0] print print 'use: %s [ option ]... [ databasefile ]...' print print ' databasefile path to a BC databasefile; default is the dbfile found in .bc_profile:' print ' %s' % default_dbfile print print 'options:' print ' -? --help print the help information and exit' print ' -D --debug turn on debugging' print ' -B --brief brief display' print ' -M --monitor turn on resource monitoring' print ' -U --update update databases with any missing post-import coefficient matrices' sys.exit(1) elif arg == '-B' or arg == '--brief': brief = True elif arg == '-D' or arg == '--debug': debug = True elif arg == '-M' or arg == '--monitor': ConciseMonitor.enable(True) elif arg == '-U' or arg == '--update': update = True elif arg == '-D' or arg == '--debug': debug = True else: Database(arg) # raise exception now if not a valid db file dbfiles.append(arg) if not dbfiles: dbfiles.append(default_dbfile) for dbfile in dbfiles: resuse = ConciseMonitor() db = Database(dbfile) report_coefficients(db,brief,update) resuse.report('%s finished' % db.basename())
#!/usr/bin/env python import re,sys from application import * from timeseries import * from coefficient import * # problems arise if you import coefficient before timeseries :( from monitor import ConciseMonitor from util import * ConciseMonitor.enable(False) class Showxy: DEFAULT_DATAPOINTS = TimeSeries.maxStringisedDatapoints def __init__(self,**args): debugpatt = re.compile('debug_') app = Application(None,"Showxy",args['debug'],False) if args['debug']: for k in args.keys(): if debugpatt.search(k): app.params.params[k] = True dbfile = args['dbfile'] or app.dbFilePath db = args['db'] = Database(dbfile) tssids = TimeSeriesSet.seriessetids(db) index = TimeSeriesSet.index(db)
def _create_all_derived_tables(cls, db): """ create and populate fresh derived tables in the database for all the derived coefficient matrices -- for all timeseries and all coeffids """ for matrixname in derived_matrix_names: tablename = derived_matrix_tablenames[matrixname] indexname = derived_matrix_indexnames[matrixname] db_drop_create_usage = ConciseMonitor() db.execute("DROP TABLE IF EXISTS %s" % tablename) db.execute("DROP INDEX IF EXISTS %s" % indexname) db.execute("CREATE TABLE %s(coeffid INTEGER, seriesid1 INTEGER, seriesid2 INTEGER, coeff REAL)" % tablename) db.execute("CREATE UNIQUE INDEX %s ON %s(coeffid,seriesid1,seriesid2)" % (indexname, tablename)) db_drop_create_usage.report('DROP AND CREATE TABLE %s and its INDEX' % tablename) ninsert = 0 for seriessetid in db.selectColumn("SELECT seriessetid FROM seriesset"): create_usage = ConciseMonitor() create_usage.report('CREATING post-import tables ONCE for database %s tssid %s' % (db.basename(), seriessetid)) o2s = { } # map timeseries' ordinal to seriesid for a given seriessetid s2o = { } # and map seriesid to ordinal ordinal = 0 for seriesid in db.selectColumn("SELECT seriesid FROM series WHERE seriessetid=? ORDER BY seriesid", seriessetid): o2s[ordinal] = seriesid s2o[seriesid] = ordinal ordinal += 1 n = len(o2s) for coeffid in db.selectColumn("SELECT coeffid FROM coeff WHERE seriessetid=?", seriessetid): coeffdict = Dict(db,"CoefficientMatrix",coeffid) try: issymm = coeffdict.getbool('issymm') # the Coefficients's dict 'issymm' tells us if the original imported coeffvalues matrix is symmetrical except: issymm = False # there are a few V4.000 databases that do not a 'issymm' entry in the dict :( maxabscoeff = 0 originals = SparseMatrix(n) db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM coeffvalue WHERE coeffid=?",[coeffid]) for (id1, id2, coeff) in db.cursor().fetchall(): if id1 not in s2o: raise Exception("seriesid1=%s not found for coeffid=%s in database %s for coeffvalue" % (id1, coefficients.coeffid(), db.basename())) if id2 not in s2o: raise Exception("seriesid2=%s not found for coeffid=%s in database %s for coeffvalue" % (id2, coefficients.coeffid(), db.basename())) if coeff: originals [s2o[id1]] [s2o[id2]] = coeff if issymm: assert s2o[id1] > s2o[id2] originals [s2o[id2]] [s2o[id1]] = coeff if abs(coeff) > maxabscoeff: maxabscoeff = abs(coeff) crumb = 0 for i in range(n): row = originals[i] for j in range(i): coeff1 = 0 if j not in row else row[j] coeff2 = 0 if i not in originals[j] else originals[j][i] if abs(coeff1) >= abs(coeff2): best, secondary, direction = coeff1, coeff2, 1 else: best, secondary, direction = coeff2, coeff1, 2 if best: norm = abs(best) / maxabscoeff id1 = o2s[i] id2 = o2s[j] db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['best'], coeffid, id1, id2, best) db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['secondary'], coeffid, id1, id2, secondary) db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['direction'], coeffid, id1, id2, direction) db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['norm'], coeffid, id1, id2, norm) ## create_usage.report('[%s] INSERT INTO {best,norm,direction...} WHERE coeffid=%s i=%s, j=%s' % (crumb, coeffid, i,j )) crumb += 4 db.commit() create_usage = create_usage.report('INSERT INTO {best,norm,direction...} WHERE coeffid=%s, %s INSERT operations tssid %s' % (coeffid,crumb,seriessetid)) ninsert += crumb create_usage.report('CREATE port-import tables complete, %s INSERT operations across all tssids' % ninsert) return crumb
def __init__(self, db=None, seriessetid=None): resource_usage = ConciseMonitor() resource_timeseries = 0 resource_coefficients = 0 if db and not seriessetid: raise Exception("cannot create a TimeSeriesSet from the database (db=%s) without a seriessetid" % db.filepath()) self.app = Application.the_app self._series = [ ] self._shared_times = None self._start = None self._interesting_categoryparts_ordinal = [ ] # each unique "first interesting" TimeSeries category part is assigned an ordinal starting at 0 self._interesting_categoryparts_level = 0 # "first interesting" means the level into the category parts that changes form one TimeSeries to the next self._events = None self._dict = None self._db = db self._seriessetid = None self._coeffids = [ ] self._coeffindex = dict() self._coeffnames = dict() self._coefficients = None self._usesparse = True if db: Dict.KNOWNCLASSES = None # HACK HACK HACK - force Dict to reload its KNOWNCLASSES since they can vary from one database to another self._dict = Dict(db,"TimeSeriesSet",seriessetid) self._seriessetid = seriessetid self._shared_times = db.selectColumn("SELECT time FROM time WHERE seriessetid=? ORDER BY timeid",seriessetid) self._start = TimePoint.fromString(TimePoint.intervalOf(self._shared_times), self._shared_times[0]) seriesids = db.selectColumn("SELECT seriesid FROM series WHERE seriessetid=?",self._seriessetid) for ordinal in range(len(seriesids)): self._series.append(TimeSeries(db,self,seriesids[ordinal],ordinal)) resource_timeseries = len(self) self._events = EventSeriesSet(self,db) # load the events only after _shared_times has been initialised self.app.log("TimeSeriesSe %s [%s series]" % (self.details(), len(self))) for row in db.selectRows("SELECT coeffid,value FROM coeff c, dict d WHERE c.seriessetid=? AND c.coeffid=d.clientid AND d.key='description' ORDER BY coeffid DESC",seriessetid): self._coeffids.append(int(row[0])) self._coeffindex[int(row[0])] = row[1] for row in db.selectRows("SELECT value,coeffid FROM coeff c, dict d WHERE c.coeffid=d.clientid AND d.key='name'"): self._coeffnames[row[0]] = row[1] self.loadcoefficients(db,self._coeffids[0]) resource_coefficients = self._coefficients.cardinality() lpi = 0 # give each different interesting (first or deeper "level" or interesting_categoryparts_level) categorypart a unique ordinal maxnparts = 0 for ts in self._series: maxnparts = max(maxnparts, len(ts.categoryparts())) for ts in self._series: ts.ensure_categoryparts(maxnparts) for _interesting_categoryparts_level in range(maxnparts): self._interesting_categoryparts_ordinal = { } # map categorypart string to ordinal int for ts in self._series: if ts.categoryparts()[self._interesting_categoryparts_level] not in self._interesting_categoryparts_ordinal: self._interesting_categoryparts_ordinal[ts.categoryparts()[_interesting_categoryparts_level]] = lpi lpi += 1 if len(self._interesting_categoryparts_ordinal): break if len(self._interesting_categoryparts_ordinal) == 0: self._interesting_categoryparts_level = 0 self.log("interesting_categoryparts_ordinal = %s" % self._interesting_categoryparts_ordinal) tssid = '#%s' % self.seriessetid() if self.seriessetid() else '' resource_usage.report('%s init %s %s ts %s coeffs' % (type(self).__name__, tssid, resource_timeseries, resource_coefficients))
class GalaxyLayoutThread(QtCore.QThread): def __init__(self, root): QtCore.QThread.__init__(self) self.root = root self.phases = [ # These don't change no matter what self.task_preamble, self.task_computeCoefficients, self.task_computeOptimalDistances, # These depend on what the central node is self.task_cutoffOuterLinks, self.task_cutoffCentralLinks, self.task_makeLinks, self.task_tupliseNodeLinks, self.task_removeUnattachedNodes, self.task_findRangeOfCentralLinks, self.task_computeSecondaryNodeIndexList, self.task_computeAlphae, self.task_computeOptimalDistancesToCenterNode, self.task_placeNodes, self.task_setupAdjust, self.task_setupAdjustAllNodes, self.task_adjustAllNodes, self.task_noLongerWorking, self.task_end, ] self.phase = self.phases[0] self.phaseIndex = 0 self.displayPhaseSignal = dict() self.phasesByName = dict() self.maxIndex = -1 self.entryByStep = False for _index, _phase in enumerate(self.phases): self.displayPhaseSignal[_phase.__name__] = QtCore.Signal() QtCore.QObject.connect( self, QtCore.SIGNAL(_phase.__name__ + "Signal ()"), _phase, QtCore.Qt.QueuedConnection ) self.phasesByName[_phase.__name__] = _index step = QtCore.Signal() QtCore.QObject.connect(self, QtCore.SIGNAL("step ()"), self.doPhase, QtCore.Qt.QueuedConnection) displayReset = QtCore.Signal() # Reset - call when new database is loaded QtCore.QObject.connect(self, QtCore.SIGNAL("displayReset ()"), self.reset, QtCore.Qt.QueuedConnection) self.quit = QtCore.Signal() QtCore.QObject.connect(self, QtCore.SIGNAL("quit ()"), self.doQuit) self.START = time.time() self.suspend = True def task_preamble(self): if not self.enterThisFunction(): return self.resource_usage = ConciseMonitor() self.root.galaxy.view.links = [] self.root.galaxy.view.visibleNodeDetails.setNodeList([self.root.centerNode()]) try: self.root.nodes[self.root.centerNode()].hiding = False except: self.root.setCenterNode(0) self.root.nodes[0].hiding = False self.nextStep() def showSparse(self, matrix, integer=False): result = "[" for i in matrix: result += "{" added = False for j in i: if integer: result += "%d:%4d" % (j, i[j]) + ", " else: result += "%d:%.2f" % (j, i[j]) + ", " added = True if added: result = result[:-2] result += "}" result += "]" return result def task_computeCoefficients(self): if not self.enterThisFunction(): return self.computeListOfVisibleNodes() self.maxCoefficient = self.root.cm.maxabscoeff() self.bestCoefficients = self.root.cm.best self.secondaryCoefficients = self.root.cm.secondary self.directions = self.root.cm.direction self.absNormalizedCoefficients = self.root.cm.norm # Create an empty vector that we'll use later self.optimalCentralLinkLength = self.makeEmptyRowVector() self.nextStep() def task_computeOptimalDistances(self): if not self.enterThisFunction(): return self.computeListOfVisibleNodes() self.optimalCrosslinkLength = [] for i in range(self.root.N): _optimalCrosslinkLength = dict() _normalizedCoefficientVector = self.absNormalizedCoefficients[i] for j in _normalizedCoefficientVector: _distance = int((1.0 - _normalizedCoefficientVector[j]) * self.root.MAX_CROSSLINK_LENGTH) if _distance < self.root.MINIMUM_SEPARATION: _distance = self.root.MINIMUM_SEPARATION _optimalCrosslinkLength[j] = _distance self.optimalCrosslinkLength.append(_optimalCrosslinkLength) self.nextStep() def task_cutoffOuterLinks(self): if not self.enterThisFunction(): return self.root.galaxy.view.outerLinkCutoffSlider.slider.value() self.root.displayedLastTime = self.root.galaxy.view.visibleNodeDetails.nodeList() _sliderValue = (255 - self.root.galaxy.view.outerLinkCutoffSlider.slider.value()) / 255.0 self.computeListOfVisibleNodes() _centerNode = self.root.centerNode() ## Value is 0 _newVisibleNodeIndexList = [] _sparse = self.root.cm.sparse ### value is true _normalizedCoefficients = self.absNormalizedCoefficients[_centerNode] for i in self.visibleNodeIndexList: if i == _centerNode: _newVisibleNodeIndexList.append(_centerNode) else: _normalized = 0 if _sparse and i not in _normalizedCoefficients else _normalizedCoefficients[i] if _normalized >= _sliderValue: _newVisibleNodeIndexList.append(i) self.visibleNodeIndexList = ( _newVisibleNodeIndexList ) ## Takes input of the point value will be plotted in cercle # print self.visibleNodeIndexList self.nextStep() def task_cutoffCentralLinks(self): _categoryData = [] parentData = [] if self.root.db: _series = self.root.tss.getseries for i in range(self.root.N): _seriesElement = _series(i) if i > 0: try: _seriesElement.category().split(":")[2] except: parentData.append(i) _categoryData.append([_seriesElement.category(), _seriesElement.label(), i]) if not self.enterThisFunction(): return self.root.displayedLastTime = self.root.galaxy.view.visibleNodeDetails.nodeList() _sliderValue = (255 - self.root.galaxy.view.centralLinkCutoffSlider.slider.value()) / 255.0 self.computeListOfVisibleNodes() _centerNode = self.root.centerNode() ## Value is 0 _newVisibleNodeIndexList = [] _sparse = self.root.cm.sparse ### value is true _normalizedCoefficients = self.absNormalizedCoefficients[_centerNode] for i in self.visibleNodeIndexList: if i == _centerNode: _newVisibleNodeIndexList.append(_centerNode) else: _normalized = 0 if _sparse and i not in _normalizedCoefficients else _normalizedCoefficients[i] if _normalized >= _sliderValue or i in parentData: _newVisibleNodeIndexList.append(i) self.visibleNodeIndexList = ( _newVisibleNodeIndexList ) ## Takes input of the point value will be plotted in cercle # print self.visibleNodeIndexList self.nextStep() def task_makeLinks(self): if not self.enterThisFunction(): return _sliderValue = (255 - self.root.galaxy.view.cutoffSlider.slider.value()) / 255.0 _centerNode = self.root.centerNode() _sparse = self.root.cm.sparse for i in self.visibleNodeIndexList: _node = self.root.nodes[i].galaxy _node.linksOut = [] _node.linksIn = [] _normalizedCoefficients = self.absNormalizedCoefficients[i] _bestCoefficients = self.bestCoefficients[i] _secondaryCoefficients = self.secondaryCoefficients[i] _directions = self.directions[i] _maxCoefficient = 1.0 if self.root.cm.maxabscoeff() == 0.0 else self.root.cm.maxabscoeff() for j in self.visibleNodeIndexList: if j < i: _normalized = 0 if _sparse and j not in _normalizedCoefficients else _normalizedCoefficients[j] if (_normalized >= _sliderValue) or (i == _centerNode) or (j == _centerNode): _best = 0 if _sparse and j not in _bestCoefficients else _bestCoefficients[j] if _best != 0: _secondary = 0 if _sparse and j not in _secondaryCoefficients else _secondaryCoefficients[j] _direction = 0 if _sparse and j not in _directions else _directions[j] if _direction == 2: _link = universe.GalaxyLink(self.root, j, i, _best, _secondary, _maxCoefficient) else: _link = universe.GalaxyLink(self.root, i, j, _best, _secondary, _maxCoefficient) self.nextStep() def task_tupliseNodeLinks(self): if not self.enterThisFunction(): return _allLinks = [] for _nodeIndex in self.visibleNodeIndexList: _node = self.root.nodes[_nodeIndex].galaxy for _link in _node.linksIn: _allLinks.append(_link) _node.linksIn = tuple(_node.linksIn) _node.linksOut = tuple(_node.linksOut) self.root.galaxy.view.links = tuple(_allLinks) self.nextStep() def task_removeUnattachedNodes(self): if not self.enterThisFunction(): return _centerNode = self.root.centerNode() _cutoffNonZero = self.root.galaxy.view.centralLinkCutoffSlider.slider.value() != 255 for _index in self.visibleNodeIndexList[:]: _node = self.root.nodes[_index] _galaxy = _node.galaxy if _index == _centerNode: for _link in _galaxy.linksIn: _link.zValue = 4 for _link in _galaxy.linksOut: _link.zValue = 4 else: if _cutoffNonZero: if len(_galaxy.linksOut): None elif len(_galaxy.linksIn): None else: self.visibleNodeIndexList.remove(_index) self.nextStep() def task_removeUnattachedNodes(self): if not self.enterThisFunction(): return _centerNode = self.root.centerNode() _cutoffNonZero = self.root.galaxy.view.outerLinkCutoffSlider.slider.value() != 255 for _index in self.visibleNodeIndexList[:]: _node = self.root.nodes[_index] _galaxy = _node.galaxy if _index == _centerNode: for _link in _galaxy.linksIn: _link.zValue = 4 for _link in _galaxy.linksOut: _link.zValue = 4 else: if _cutoffNonZero: if len(_galaxy.linksOut): None elif len(_galaxy.linksIn): None else: self.visibleNodeIndexList.remove(_index) self.nextStep() def task_findRangeOfCentralLinks(self): coffList = [] if not self.enterThisFunction(): return _minCoeff = 1.0e99 _maxCoeff = 0.0 _centerNode = self.root.centerNode() _bestCoefficients = self.bestCoefficients[_centerNode] _sparse = self.root.cm.sparse newCoeff = 1 for _nodeIndex in self.visibleNodeIndexList: if _nodeIndex != _centerNode: _coeff = 0 if _sparse and _nodeIndex not in _bestCoefficients else abs(_bestCoefficients[_nodeIndex]) _maxCoeff = max(_maxCoeff, _coeff) _minCoeff = min(_minCoeff, _coeff) coffList.append(_coeff) # print _coeff # print _nodeIndex if _coeff < newCoeff: newCoeff = _coeff outerNode = _nodeIndex ## much anticipated coefficient # Calculate the scaling values # print 'min',newCoeff," node",outerNode sortedCoffList = sorted(coffList) # print sortedCoffList # print sortedCoffList self.coeffShift = _minCoeff try: self.coeffMultiplier = 1.0 / (_maxCoeff - _minCoeff) except: self.coeffMultiplier = 0.0 self.coeffShift = 0.0 self.secondaryNodeIndexList = self.visibleNodeIndexList[:] # Remove the central node from the list of secondary nodes if _centerNode in self.secondaryNodeIndexList: self.secondaryNodeIndexList.remove(_centerNode) if _centerNode not in self.visibleNodeIndexList: print "The central node appears to be unapparent" self.nextStep() def task_computeSecondaryNodeIndexList(self): if not self.enterThisFunction(): return _list = self.visibleNodeIndexList[:] _filteredList = [] # ANDY 2011-11-04 Added this line self.secondaryNodeIndexList = self.visibleNodeIndexList[:] """ for _nodeIndex in _list: _node = self.root.nodes [_nodeIndex] if (len (_node.galaxy.linksIn) + len (_node.galaxy.linksOut)) > 0: _filteredList.append (_nodeIndex) self.secondaryNodeIndexList = _filteredList """ self.nextStep() def task_computeAlphae(self): if not self.enterThisFunction(): return self.alpha = dict() # Dictionary of bearings if self.root.centerNode() in self.secondaryNodeIndexList: self.secondaryNodeIndexList.remove(self.root.centerNode()) if len(self.secondaryNodeIndexList): for _nodeIndex in self.visibleNodeIndexList: self.alpha[_nodeIndex] = self.Talpha(self.root.centerNode(), self.secondaryNodeIndexList[0], _nodeIndex) self.nextStep() def task_computeOptimalDistancesToCenterNode(self): #### Here we are calculating the length which value should be plotted where if not self.enterThisFunction(): return self.root.galaxy.view.visibleNodeDetails.setNodeList(self.visibleNodeIndexList) _power = (1.0 + (self.root.galaxy.view.centrifugeSlider.slider.value())) / 256.0 _centerNode = self.root.centerNode() _bestCoefficients = self.bestCoefficients[_centerNode] _sparse = self.root.cm.sparse for _nodeIndex in self.visibleNodeIndexList: if _nodeIndex != _centerNode: _best = 0 if _sparse and _nodeIndex not in _bestCoefficients else abs(_bestCoefficients[_nodeIndex]) _coeff = (_best - self.coeffShift) * self.coeffMultiplier _centrifugedCoefficient = 1.0 - math.pow(_coeff, _power) self.optimalCentralLinkLength[_nodeIndex] = ( _centrifugedCoefficient * self.root.SCALING + self.root.OFFSET ) # print self.optimalCentralLinkLength [_nodeIndex] length = self.optimalCentralLinkLength[_nodeIndex] # print length,"node ",_nodeIndex self.nextStep() def computeListOfVisibleNodes(self): self.visibleNodeIndexList = [] for _nodeIndex, _node in enumerate(self.root.nodes): if not _node.hiding: self.visibleNodeIndexList.append(_nodeIndex) if not self.root.centerNode() in self.visibleNodeIndexList: self.visibleNodeIndexList.append(self.root.centerNode()) def task_placeNodes(self): if not self.enterThisFunction(): return # Reset the node position information self.root.galaxy.view.visibleNodeDetails.resetPositions() # Place the primary node self.root.galaxy.view.visibleNodeDetails.setPosition(self.root.centerNode(), QtCore.QPointF(0, 0)) # Place non-central nodes if len(self.secondaryNodeIndexList): self.secondaryNode = self.secondaryNodeIndexList[0] for _nodeIndex in self.secondaryNodeIndexList: _currentPosition = self.root.nodes[_nodeIndex].galaxy.previousEndpoint try: self.alpha[_nodeIndex] = math.atan(_currentPosition.y() / _currentPosition.x()) if _currentPosition.x() < 0.0: self.alpha[_nodeIndex] -= math.pi except: self.alpha[_nodeIndex] = random.uniform( 0.0, 2.0 * math.pi ) # Start with nodes with random alphae so when crosslink cutoff is max we have some spread _intensity = self.distance(_nodeIndex) _alpha = self.alpha[_nodeIndex] self.root.galaxy.view.visibleNodeDetails.setPosition( _nodeIndex, QtCore.QPointF(_intensity * math.cos(_alpha), _intensity * math.sin(_alpha)) ) # print self.visibleNodeIndexList self.nextStep() def task_setupAdjust(self): if not self.enterThisFunction(): return self.DEGREES = math.pi / 180.0 # Usual values self.alphaDelta = 130.0 * self.DEGREES self.alphaDeltaReductionFactor = 0.7 self.alphaDeltaSmallest = 5.0 * self.DEGREES self.nextStep() def task_setupAdjustAllNodes(self): # See how many iterations we need at most - this is just so we can indicate progress # self.iterations = int (math.log10 (self.alphaDeltaSmallest / self.alphaDelta) / math.log10 (self.alphaDeltaReductionFactor)) + 1 self.iteration = 0 self.thisNode = 1 # This has to start at 1, not zero. We ignore the first value in the list self.alphaDelta = 130.0 * self.DEGREES if len(self.secondaryNodeIndexList): self.lastNode = self.secondaryNodeIndexList[-1] self.changesThisIteration = False self.nextStep() else: # Skip the next step self.phase = self.phases[self.phases.index(self.phase) + 1] self.nextStep() def task_adjustAllNodes(self): if not self.enterThisFunction(): return # Don't want to reset timer _startTime = time.time() self.DECLUTTER_THRESHOLD = 400 # This is the square of the repulsion distance self.declutterSliderValue = self.root.galaxy.view.declutterSlider.slider.value() self.lsZeroCache = [] _secondaryNodeIndexList = self.secondaryNodeIndexList _alphaDeltaReductionFactor = self.alphaDeltaReductionFactor self.lsZeroCache = [-1] * self.root.N if len(_secondaryNodeIndexList) > 1: _thisNode = _secondaryNodeIndexList[self.thisNode] while (time.time() - _startTime) < 0.01: self.changesMade = self.lsIterationOnOneNode(_thisNode, _secondaryNodeIndexList, self.alphaDelta) self.changesThisIteration = True if self.alphaDelta < self.alphaDeltaSmallest: self.nextStep() return else: if _thisNode == self.lastNode: if self.changesThisIteration: self.thisNode = 1 _thisNode = _secondaryNodeIndexList[self.thisNode] self.alphaDelta *= _alphaDeltaReductionFactor self.iteration += 1 self.changesThisIteration = False else: self.nextStep() return else: self.thisNode += 1 _thisNode = _secondaryNodeIndexList[self.thisNode] self.repeatStep() else: self.nextStep() def task_noLongerWorking(self): if not self.enterThisFunction(): return self.nextStep() def task_end(self): if not self.enterThisFunction(): return def makeEmptyRowVector(self): _vector = [] for i in range(self.root.N): _vector.append(0) return _vector def makeEmptyTriangularMatrix(self): _matrix = [] for i in range(self.root.N): _row = [] for j in range(i): _row.append(0) _matrix.append(_row) return _matrix def distance(self, j): return self.optimalCentralLinkLength[j] def suspendRedraw(self, state): self.suspend = state def getOptimalCrosslinkLength(self, nodeIndex1, nodeIndex2): _lengths = self.optimalCrosslinkLength if nodeIndex2 in _lengths[nodeIndex1]: return _lengths[nodeIndex1][nodeIndex2] else: return self.root.MAX_CROSSLINK_LENGTH def Talpha(self, centralNodeIndex, nodeIndex1, nodeIndex2): if nodeIndex1 == nodeIndex2: return 3.14 elif nodeIndex1 == centralNodeIndex: return 3.14 elif nodeIndex2 == centralNodeIndex: return 3.14 _a = self.distance(nodeIndex1) _b = self.distance(nodeIndex2) _d = self.root.galaxyLayoutThread.getOptimalCrosslinkLength(nodeIndex1, nodeIndex2) _a2 = _a * _a _b2 = _b * _b _d2 = _d * _d try: result = math.acos((_a2 + _b2 - _d2) / (2.0 * _a * _b)) except: result = 3.14 return result def placeNode(self, centralNodeIndex, thisNodeIndex): _optimalDistance = self.distance(thisNodeIndex) _x = _optimalDistance * math.cos(self.alpha[thisNodeIndex]) _y = _optimalDistance * math.sin(self.alpha[thisNodeIndex]) self.root.galaxy.view.visibleNodeDetails.setPosition(thisNodeIndex, QtCore.QPointF(_x, _y)) def sumOfSquares(self, centralNodeIndex, thisNode, _listOfVisibleNodes): sum = 0 _nodeIterationList = _listOfVisibleNodes[:] if thisNode in _nodeIterationList: _nodeIterationList.remove(thisNode) posThis = self.root.galaxy.view.visibleNodeDetails.position(thisNode) posThisX = posThis.x() posThisY = posThis.y() _sliderValue = (255 - self.root.galaxy.view.cutoffSlider.slider.value()) / 255.0 _normalizedCoefficients = self.absNormalizedCoefficients[thisNode] _sparse = self.root.cm.sparse for nodeIndex in _nodeIterationList: _normalized = ( 0 if _sparse and nodeIndex not in _normalizedCoefficients else _normalizedCoefficients[nodeIndex] ) if _normalized < _sliderValue: continue posIndex = self.root.galaxy.view.visibleNodeDetails.position(nodeIndex) distanceX = posThisX - posIndex.x() distanceY = posThisY - posIndex.y() # Make squares by multiplying numbers by themselves. Don't use math.pow: it's way too slow effect = distanceX * distanceX + distanceY * distanceY _term = self.getOptimalCrosslinkLength(thisNode, nodeIndex) optimal = _term * _term _diff = abs(effect - optimal) if effect < self.DECLUTTER_THRESHOLD: _power = 1.0 + self.declutterSliderValue / 500.0 _addition = math.pow( _diff * ( 1 + ( (self.declutterSliderValue / 255.0) * (self.DECLUTTER_THRESHOLD - effect) / self.DECLUTTER_THRESHOLD ) ), _power, ) sum += _addition else: sum += _diff return sum def thisFunction(self): return inspect.stack()[2][3] def enterThisFunction(self): if self.entryByStep: self.entryByStep = False else: self.BIG_START = time.time() self.root.galaxyTime = ConciseMonitor() self.START = time.time() self.root.galaxy.view.lastTimeDisplayed = time.time() _requestedFunction = self.thisFunction() _requestedIndex = self.phasesByName[_requestedFunction] _currentIndex = self.phasesByName[self.phase.__name__] if _currentIndex > self.maxIndex: self.maxIndex = _currentIndex if _requestedIndex > self.maxIndex: return False else: self.phase = self.phases[_requestedIndex] return True def makeListOfVisibleNodes(self): if not self.enterThisFunction(): return self.computeListOfVisibleNodes() self.nextStep() def nextStep(self): self.resource_usage.report("nextStep: %s" % (self.phase.__name__)) self.resource_usage = ConciseMonitor() # Report how long the last phase took ##print self.phase.__name__ if self.phase.__name__ == "task_preamble": self.START = time.time() self.BIG_START = self.START self.START = self.root.INSTRUMENT_CENTRIFUGE(self.phase.__name__, self.START) ##print self.START self.iteration = 0 # prepare for next phase self.phase = self.phases[self.phases.index(self.phase) + 1] # Tell the GUI thread to move on self.root.galaxy.view.display.emit() def repeatStep(self): self.root.galaxy.view.display.emit() def reset(self): self.phaseIndex = 0 self.phase = self.phases[0] self.iteration = 0 self.maxIndex = -1 self.entryByStep = False def doPhase(self): if not self.suspend: if self.iteration == 0: self.START = time.time() self.entryByStep = True self.phase() def doQuit(self): None def run(self): self.setPriority(QtCore.QThread.LowestPriority) self.exec_() # Wait for events directed to this thread def lsIterationOnOneNode(self, nodeIndex, nodeIndexList, alphaDelta): if self.lsZeroCache[nodeIndex] == -1: lsZero = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList) else: lsZero = self.lsZeroCache[nodeIndex] _initialAlpha = self.alpha[nodeIndex] self.alpha[nodeIndex] += alphaDelta self.placeNode(self.root.centerNode(), nodeIndex) lsPlus = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList) if lsPlus >= lsZero: self.alpha[nodeIndex] = _initialAlpha - alphaDelta self.placeNode(self.root.centerNode(), nodeIndex) lsMinus = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList) if lsMinus >= lsZero: self.alpha[nodeIndex] = _initialAlpha self.placeNode(self.root.centerNode(), nodeIndex) return False else: self.lsZeroCache[nodeIndex] = lsMinus else: self.lsZeroCache[nodeIndex] = lsPlus return True
def time_one(db, tssid): keep = list() # keep is required to prevent python GC from deleting unused tss and coeff matrices for usesparse in (False,True): resuse = ConciseMonitor() tss = TimeSeriesSet(db,tssid) # ,usesparse) tss._usesparse = usesparse keep.append(tss) cindex = tss.coeffindex() # dict of { coeffid, description } for all coefficient matrices keep.append(tss.coefficients()) for coeffid in sorted(cindex.keys()): localuse = ConciseMonitor() cm = tss.loadcoefficients(db,coeffid) keep.append(cm) localuse.report('load %10.10s %30.30s cardinality=%s' % (type(cm).__name__, cm.description(), cm.cardinality())) sumuse = ConciseMonitor() coeffs, n = cm.readyloop() sum = 0 if cm.sparse: for i in range(n): row = coeffs[i] for j in row: sum += row[j] else: for i in range(n): row = coeffs[i] for j in range(n): sum += row[j] sumuse = sumuse.report('sum of all %s cells = %s explicit' % (n*n, sum)) if cm.sparse: coeffs, n = cm.readyloop() sum = 0 for i in range(n): for j in coeffs[i]: sum += coeffs.get(i,j) sumuse = sumuse.report('sum of all %s cells = %s using Sparse.get(i,j)' % (n*n, sum)) sumuse = sumuse.report('sum of all %s cells = %s implicit by row' % (n*n, cm.sum_by_rows())) sumuse.report('sum of all %s cells = %s implicit by col' % (n*n, cm.sum_by_cols())) resuse.report('test_one %s ^^^^^^^^^^^^^^^^^' % cm.matrix_type())
def time_nested_loop(n): loopuse = ConciseMonitor() for i in range(n): for j in range(n): pass loopuse.report('loopuse - empty nested loop of %s x %s = %s iterations' % (n,n,n*n))