def __init__ (self, tss, centralNodeId, centralNode, normalised = False, parent = None, visibleOnly = True, *args):
resource_usage = ConciseMonitor()
QDialog.__init__ (self, *args)
self.tss = tss
self.centralNodeId = centralNodeId
self.centralNode = centralNode
self.normalised = normalised
self.parent = parent
self.visibleOnly = visibleOnly
self.setAttribute (Qt.WA_DeleteOnClose, True)
self.setWindowTitle ('Coefficient List')
self.setMinimumSize (QSize (700, 400))
_layout = QVBoxLayout (self)
self.view = QTableView (self)
_layout.addWidget (self.view)
self.model = ProfileModel (tss, centralNodeId, centralNode, normalised, parent, visibleOnly)
self.view.setModel (self.model)
# hide grid
#self.view.setShowGrid (False)
# sorting
self.proxyModel = QSortFilterProxyModel ()
self.proxyModel.setSourceModel (self.model)
self.proxyModel.setDynamicSortFilter (True)
self.view.setModel (self.proxyModel)
self.view.setSortingEnabled (False)
self.view.horizontalHeader ().setSortIndicatorShown (False)
self.view.horizontalHeader ().setClickable (False)
self.view.setColumnHidden (0, True)
self.view.setColumnHidden (1, True)
# set column width to fit contents
self.view.resizeColumnsToContents ()
# Add a status line
self.statusLine = QLabel (self.reportStatistics ())
_layout.addWidget (self.statusLine)
# Add buttons
_buttonLayout = QHBoxLayout ()
_layout.addLayout (_buttonLayout)
self.allButton = QPushButton ('All nodes')
_buttonLayout.addWidget (self.allButton)
QObject.connect (self.allButton, SIGNAL ('clicked ()'), self.allNodes)
self.visibleButton = QPushButton ('Visible nodes only')
_buttonLayout.addWidget (self.visibleButton)
QObject.connect (self.visibleButton, SIGNAL ('clicked ()'), self.visibleNodes)
self.okButton = QPushButton ('OK')
_buttonLayout.addWidget (self.okButton)
QObject.connect (self.okButton, SIGNAL ('clicked ()'), self.ok)
resource_usage.report('CoefficientsProfile init')
self.view.showRow (0)
self.sortIt ()
def time_minimum(n):
loopuse = ConciseMonitor()
for i in range(n):
for j in range(n):
k = i
if i < j:
k = j
loopuse = loopuse.report('loopuse - loop of i<j for minimum: %s x %s = %s iterations' % (n,n,n*n))
for i in range(n):
for j in range(n):
k = min(i,j)
loopuse = loopuse.report('loopuse - loop of min() for minimum: %s x %s = %s iterations' % (n,n,n*n))
def tunit(verbose):
ConciseMonitor.enable(False)
app = Application(None,"TestTimeSeries",verbose,True)
db = Database(app.dbFilePath)
if verbose: print >>sys.stderr,'TimeSeriesSet.uniqueid for %s is %s' % (db.basename(),TimeSeriesSet.uniqueid(db))
tssids = TimeSeriesSet.seriessetids(db)
if not len(tssids): raise Exception, "there are no TimeSeriesSets in %s" % db.filepath()
tss = TimeSeriesSet(db,tssids[0])
tss.lastreldelta_quintiles()
evs = tss.events()
print "TimeSeries and TimeSeriesSet okay"
print "EventSeries and EventSeriesSet okay"
def __init__(self, links, scaling=1):
resource_usage = ConciseMonitor()
self._scaling = scaling
self._size = len(links)
self._links = links
# initialise the igraph.Graph
_ig = igraph.Graph(self._size)
_weights = []
for i in range(self._size):
for j in range(i):
_ig.add_edges((i, j))
_corr = self.getLink(i, j)
_w = int(1000 * _corr)
_weights.append(_w)
resource_usage.report(' FruchtermanReingold: weights')
x = []
for i in range(self._size):
_alpha = i * math.pi * 2 / self._size
x.append(10 * [math.cos(_alpha), 10 * math.sin(_alpha)])
resource_usage.report(' FruchtermanReingold: initial positions')
_coordinates = self.recentre(
_ig.layout_fruchterman_reingold(
seed=x, weights=_weights, repulserad=10))
resource_usage.report(' FruchtermanReingold: recentre')
_maxValue = 0.0
"""
for i in range (len (_coordinates)):
if abs (_coordinates [i] [0]) > _maxValue:
_maxValue = abs (_coordinates [i] [0])
if abs (_coordinates [i] [1]) > _maxValue:
_maxValue = abs (_coordinates [i] [0])
"""
self._nodes = []
_maxValue = 0
if _maxValue == 0:
for i in range(len(_coordinates)):
_p = QPointF(_coordinates[i][0] * scaling,
_coordinates[i][1] * scaling)
self._nodes.append(_p)
else:
for i in range(len(_coordinates)):
_p = QPointF(_coordinates[i][0] * scaling / _maxValue,
_coordinates[i][1] * scaling / _maxValue)
self._nodes.append(_p)
resource_usage.report(
' FruchtermanReingold on %d nodes complete' % (self._size))
def main():
clargs = dict(debug=False, norm=True, real=True, dbfile=None, n=None, coeff=False, intensities=False, RFORMAT=False, all=False, listindex=False, seriessetid=None, orphan=False, brief=False);
ai = 1
while ai < len(sys.argv):
arg = sys.argv[ai]
ai += 1
if arg == "-?" or arg == "--help":
Showxy.help()
sys.exit(1)
elif arg == "-A" or arg == "--all":
clargs['all'] = True
clargs['n'] = 1000000
elif arg == "-C" or arg == "--coeff":
clargs['coeff'] = True
elif arg == "-D" or arg == "--debug":
clargs['debug'] = True
elif arg == "-E" or arg == "--export":
clargs['export'] = True
elif arg == "-i" or arg == "--index":
clargs['listindex'] = True
elif arg == "-I" or arg == "--intensities":
clargs['intensities'] = True
elif arg == "-t" or arg == "--tssid":
if ai >= len(sys.argv): raise Exception("%s missing n: try --index to see an index of the database and a list of the tssids" % arg)
clargs['seriessetid'] = int(sys.argv[ai])
ai += 1
elif arg == "-M" or arg == "--monitor":
ConciseMonitor.enable(True)
elif arg == "--n" or arg == "--nonorm":
clargs['norm'] = False
elif arg == "-o" or arg == "--brieforphan":
clargs['orphan'] = True
clargs['brief'] = True
elif arg == "-O" or arg == "--orphan":
clargs['orphan'] = True
clargs['brief'] = False
elif arg == "--RFORMAT":
clargs['RFORMAT'] = True
elif arg == "--r" or arg == "--noreal":
clargs['real'] = False
elif clargs['dbfile'] == None:
clargs['dbfile'] = arg
elif clargs['n'] == None:
clargs['n'] = int(arg)
else:
raise Exception, "unrecognised arg: %s" % arg
showxy = Showxy(**clargs)
def __init__(self, matrixname, coefficients):
init__usage = ConciseMonitor()
db = coefficients.db()
tss = coefficients.tss()
super(CoefficientMatrix,self).__init__(len(tss))
self._matrixname = matrixname # matrixname is one of the names in matrix_names
# if matrixname != imported_matrix_name, the matrix is called a derived matrix
# and as such will be created on demand by this constructor if it is not
# found in the database
try:
db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM %s WHERE coeffid=?" % matrix_tablenames[matrixname],[coefficients.coeffid()])
except:
assert matrixname != imported_matrix_name
self._create_all_derived_tables(db)
db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM %s WHERE coeffid=?" % derived_matrix_tablenames[matrixname], [coefficients.coeffid()])
cardinality = 0
maxabscoeff = 0
s2o = tss.seriesid2ordinal()
if matrixname == 'value':
issymm = coefficients.values_issymmetrical() # the original imported value matrix may or may not be symmetrical
else:
issymm = True # all derived matrices are symmetrical
for (id1, id2, coeff) in db.cursor().fetchall():
if id1 not in s2o: raise Exception, "seriesid1=%s not found for coeffid=%s in database %s for matrix %s" % (id1, coefficients.coeffid(), db.basename(), self._matrixname)
if id2 not in s2o: raise Exception, "seriesid2=%s not found for coeffid=%s in database %s for matrix %s" % (id2, coefficients.coeffid(), db.basename(), self._matrixname)
if coeff:
cardinality += 1
self [s2o[id1]] [s2o[id2]] = coeff
if issymm:
assert s2o[id1] > s2o[id2]
cardinality += 1
self [s2o[id2]] [s2o[id1]] = coeff
if abs(coeff) > maxabscoeff:
maxabscoeff = abs(coeff)
self._cardinality = cardinality
self._fill = cardinality / float(self.N*self.N)
self._issymmetrical = issymm
self._maxabscoeff = maxabscoeff
init__usage.report('%s %s init %s non-zero coeffs %.2f%% mac=%s' % ('CoefficientMatrix', self.matrixname(), self.cardinality(), self.fill(), self.mac()))
def __init__ (self, tss, thisNode, normalised = False, parent = None, visibleOnly = True, *args):
resource_usage = ConciseMonitor()
QDialog.__init__ (self, *args)
self.tss = tss
self.thisNode = thisNode
self.normalised = normalised
self.parent = parent
self.visibleOnly = visibleOnly
self.setAttribute (Qt.WA_DeleteOnClose, True)
self.setWindowTitle ('Coefficients')
self.setMinimumSize (QSize (700, 400))
_layout = QVBoxLayout (self)
_currentRow = thisNode.index
self.view = QTableView (self)
_layout.addWidget (self.view)
self.model = ProfileModel (tss, thisNode, normalised, parent, visibleOnly)
self.view.setModel (self.model)
# hide grid
#self.view.setShowGrid (False)
# set column width to fit contents
#self.view.resizeColumnsToContents ()
# Colour the current row
self.view.selectRow (_currentRow)
self.view.showRow (_currentRow)
# Add a status line for Rick
self.statusLine = QLabel (self.reportStatistics ())
_layout.addWidget (self.statusLine)
# Add buttons
_buttonLayout = QHBoxLayout ()
_layout.addLayout (_buttonLayout)
self.allButton = QPushButton ('All nodes')
_buttonLayout.addWidget (self.allButton)
QObject.connect (self.allButton, SIGNAL ('clicked ()'), self.allNodes)
self.visibleButton = QPushButton ('Visible nodes only')
_buttonLayout.addWidget (self.visibleButton)
QObject.connect (self.visibleButton, SIGNAL ('clicked ()'), self.visibleNodes)
self.okButton = QPushButton ('OK')
_buttonLayout.addWidget (self.okButton)
QObject.connect (self.okButton, SIGNAL ('clicked ()'), self.ok)
resource_usage.report('CoefficientsProfile init')
def __init__(self, db, tss, coeffid, sparse=False):
resource_usage = ConciseMonitor()
self.sparse = sparse
self._N = len(tss)
self._db = db
self._tss = tss
self._coeffid = coeffid
self._dict = Dict(db,"CoefficientMatrix",coeffid) # legacy: CoefficientMatrix now refers to Coefficients in the dict for this class
self._matrices = dict() # all matrices: key is coefficient matrix name
# public attributes: each of the named CoefficientMatrixs contained in this Coefficients object
self.value = self._load('value') # original imported coefficients
self.coeffs = self.value # legacy
self.best = self._load('best') # best coefficients
self.secondary = self._load('secondary') # secondary coefficients
self.direction = self._load('direction') # direction coefficients
self.norm = self._load('norm') # normalised coefficients
resource_usage.report('%s #%s init %s matrices, values.cardinality=%s .fill=%.2f%% .mac=%s' % ('Coefficients', coeffid, len(matrix_names), self.cardinality(), 100*self.fill(), self.maxabscoeff()))
def task_preamble(self):
if not self.enterThisFunction():
return
self.resource_usage = ConciseMonitor()
self.root.galaxy.view.links = []
self.root.galaxy.view.visibleNodeDetails.setNodeList([self.root.centerNode()])
try:
self.root.nodes[self.root.centerNode()].hiding = False
except:
self.root.setCenterNode(0)
self.root.nodes[0].hiding = False
self.nextStep()
def nextStep(self):
self.resource_usage.report("nextStep: %s" % (self.phase.__name__))
self.resource_usage = ConciseMonitor()
# Report how long the last phase took
##print self.phase.__name__
if self.phase.__name__ == "task_preamble":
self.START = time.time()
self.BIG_START = self.START
self.START = self.root.INSTRUMENT_CENTRIFUGE(self.phase.__name__, self.START)
##print self.START
self.iteration = 0
# prepare for next phase
self.phase = self.phases[self.phases.index(self.phase) + 1]
# Tell the GUI thread to move on
self.root.galaxy.view.display.emit()
def utility():
ConciseMonitor.enable(False)
brief = False
debug = False
update = False
default_dbfile = Application(None,"CoefficientMatrix Utility",False,True).dbFilePath
dbfiles = [ ]
for arg in sys.argv[1:]:
if arg == '--help' or arg == '-?':
print '%s: display coefficient matrix metrics in one or more BC database files, optionally installing post-import coefficient matrices as well' % sys.argv[0]
print
print 'use: %s [ option ]... [ databasefile ]...'
print
print ' databasefile path to a BC databasefile; default is the dbfile found in .bc_profile:'
print ' %s' % default_dbfile
print
print 'options:'
print ' -? --help print the help information and exit'
print ' -D --debug turn on debugging'
print ' -B --brief brief display'
print ' -M --monitor turn on resource monitoring'
print ' -U --update update databases with any missing post-import coefficient matrices'
sys.exit(1)
elif arg == '-B' or arg == '--brief':
brief = True
elif arg == '-D' or arg == '--debug':
debug = True
elif arg == '-M' or arg == '--monitor':
ConciseMonitor.enable(True)
elif arg == '-U' or arg == '--update':
update = True
elif arg == '-D' or arg == '--debug':
debug = True
else:
Database(arg) # raise exception now if not a valid db file
dbfiles.append(arg)
if not dbfiles:
dbfiles.append(default_dbfile)
for dbfile in dbfiles:
resuse = ConciseMonitor()
db = Database(dbfile)
report_coefficients(db,brief,update)
resuse.report('%s finished' % db.basename())
#!/usr/bin/env python
import re,sys
from application import *
from timeseries import *
from coefficient import * # problems arise if you import coefficient before timeseries :(
from monitor import ConciseMonitor
from util import *
ConciseMonitor.enable(False)
class Showxy:
DEFAULT_DATAPOINTS = TimeSeries.maxStringisedDatapoints
def __init__(self,**args):
debugpatt = re.compile('debug_')
app = Application(None,"Showxy",args['debug'],False)
if args['debug']:
for k in args.keys():
if debugpatt.search(k):
app.params.params[k] = True
dbfile = args['dbfile'] or app.dbFilePath
db = args['db'] = Database(dbfile)
tssids = TimeSeriesSet.seriessetids(db)
index = TimeSeriesSet.index(db)
def _create_all_derived_tables(cls, db):
""" create and populate fresh derived tables in the database for all the derived coefficient matrices -- for all timeseries and all coeffids """
for matrixname in derived_matrix_names:
tablename = derived_matrix_tablenames[matrixname]
indexname = derived_matrix_indexnames[matrixname]
db_drop_create_usage = ConciseMonitor()
db.execute("DROP TABLE IF EXISTS %s" % tablename)
db.execute("DROP INDEX IF EXISTS %s" % indexname)
db.execute("CREATE TABLE %s(coeffid INTEGER, seriesid1 INTEGER, seriesid2 INTEGER, coeff REAL)" % tablename)
db.execute("CREATE UNIQUE INDEX %s ON %s(coeffid,seriesid1,seriesid2)" % (indexname, tablename))
db_drop_create_usage.report('DROP AND CREATE TABLE %s and its INDEX' % tablename)
ninsert = 0
for seriessetid in db.selectColumn("SELECT seriessetid FROM seriesset"):
create_usage = ConciseMonitor()
create_usage.report('CREATING post-import tables ONCE for database %s tssid %s' % (db.basename(), seriessetid))
o2s = { } # map timeseries' ordinal to seriesid for a given seriessetid
s2o = { } # and map seriesid to ordinal
ordinal = 0
for seriesid in db.selectColumn("SELECT seriesid FROM series WHERE seriessetid=? ORDER BY seriesid", seriessetid):
o2s[ordinal] = seriesid
s2o[seriesid] = ordinal
ordinal += 1
n = len(o2s)
for coeffid in db.selectColumn("SELECT coeffid FROM coeff WHERE seriessetid=?", seriessetid):
coeffdict = Dict(db,"CoefficientMatrix",coeffid)
try:
issymm = coeffdict.getbool('issymm') # the Coefficients's dict 'issymm' tells us if the original imported coeffvalues matrix is symmetrical
except:
issymm = False # there are a few V4.000 databases that do not a 'issymm' entry in the dict :(
maxabscoeff = 0
originals = SparseMatrix(n)
db.cursor().execute("SELECT seriesid1,seriesid2,coeff FROM coeffvalue WHERE coeffid=?",[coeffid])
for (id1, id2, coeff) in db.cursor().fetchall():
if id1 not in s2o:
raise Exception("seriesid1=%s not found for coeffid=%s in database %s for coeffvalue" % (id1, coefficients.coeffid(), db.basename()))
if id2 not in s2o:
raise Exception("seriesid2=%s not found for coeffid=%s in database %s for coeffvalue" % (id2, coefficients.coeffid(), db.basename()))
if coeff:
originals [s2o[id1]] [s2o[id2]] = coeff
if issymm:
assert s2o[id1] > s2o[id2]
originals [s2o[id2]] [s2o[id1]] = coeff
if abs(coeff) > maxabscoeff:
maxabscoeff = abs(coeff)
crumb = 0
for i in range(n):
row = originals[i]
for j in range(i):
coeff1 = 0 if j not in row else row[j]
coeff2 = 0 if i not in originals[j] else originals[j][i]
if abs(coeff1) >= abs(coeff2):
best, secondary, direction = coeff1, coeff2, 1
else:
best, secondary, direction = coeff2, coeff1, 2
if best:
norm = abs(best) / maxabscoeff
id1 = o2s[i]
id2 = o2s[j]
db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['best'],
coeffid, id1, id2, best)
db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['secondary'],
coeffid, id1, id2, secondary)
db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['direction'],
coeffid, id1, id2, direction)
db.execute_sans_commit("INSERT INTO %s(coeffid,seriesid1,seriesid2,coeff) VALUES(?,?,?,?)" % derived_matrix_tablenames['norm'],
coeffid, id1, id2, norm)
## create_usage.report('[%s] INSERT INTO {best,norm,direction...} WHERE coeffid=%s i=%s, j=%s' % (crumb, coeffid, i,j ))
crumb += 4
db.commit()
create_usage = create_usage.report('INSERT INTO {best,norm,direction...} WHERE coeffid=%s, %s INSERT operations tssid %s' % (coeffid,crumb,seriessetid))
ninsert += crumb
create_usage.report('CREATE port-import tables complete, %s INSERT operations across all tssids' % ninsert)
return crumb
def __init__(self, db=None, seriessetid=None):
resource_usage = ConciseMonitor()
resource_timeseries = 0
resource_coefficients = 0
if db and not seriessetid: raise Exception("cannot create a TimeSeriesSet from the database (db=%s) without a seriessetid" % db.filepath())
self.app = Application.the_app
self._series = [ ]
self._shared_times = None
self._start = None
self._interesting_categoryparts_ordinal = [ ] # each unique "first interesting" TimeSeries category part is assigned an ordinal starting at 0
self._interesting_categoryparts_level = 0 # "first interesting" means the level into the category parts that changes form one TimeSeries to the next
self._events = None
self._dict = None
self._db = db
self._seriessetid = None
self._coeffids = [ ]
self._coeffindex = dict()
self._coeffnames = dict()
self._coefficients = None
self._usesparse = True
if db:
Dict.KNOWNCLASSES = None # HACK HACK HACK - force Dict to reload its KNOWNCLASSES since they can vary from one database to another
self._dict = Dict(db,"TimeSeriesSet",seriessetid)
self._seriessetid = seriessetid
self._shared_times = db.selectColumn("SELECT time FROM time WHERE seriessetid=? ORDER BY timeid",seriessetid)
self._start = TimePoint.fromString(TimePoint.intervalOf(self._shared_times), self._shared_times[0])
seriesids = db.selectColumn("SELECT seriesid FROM series WHERE seriessetid=?",self._seriessetid)
for ordinal in range(len(seriesids)):
self._series.append(TimeSeries(db,self,seriesids[ordinal],ordinal))
resource_timeseries = len(self)
self._events = EventSeriesSet(self,db) # load the events only after _shared_times has been initialised
self.app.log("TimeSeriesSe %s [%s series]" % (self.details(), len(self)))
for row in db.selectRows("SELECT coeffid,value FROM coeff c, dict d WHERE c.seriessetid=? AND c.coeffid=d.clientid AND d.key='description' ORDER BY coeffid DESC",seriessetid):
self._coeffids.append(int(row[0]))
self._coeffindex[int(row[0])] = row[1]
for row in db.selectRows("SELECT value,coeffid FROM coeff c, dict d WHERE c.coeffid=d.clientid AND d.key='name'"):
self._coeffnames[row[0]] = row[1]
self.loadcoefficients(db,self._coeffids[0])
resource_coefficients = self._coefficients.cardinality()
lpi = 0 # give each different interesting (first or deeper "level" or interesting_categoryparts_level) categorypart a unique ordinal
maxnparts = 0
for ts in self._series:
maxnparts = max(maxnparts, len(ts.categoryparts()))
for ts in self._series:
ts.ensure_categoryparts(maxnparts)
for _interesting_categoryparts_level in range(maxnparts):
self._interesting_categoryparts_ordinal = { } # map categorypart string to ordinal int
for ts in self._series:
if ts.categoryparts()[self._interesting_categoryparts_level] not in self._interesting_categoryparts_ordinal:
self._interesting_categoryparts_ordinal[ts.categoryparts()[_interesting_categoryparts_level]] = lpi
lpi += 1
if len(self._interesting_categoryparts_ordinal):
break
if len(self._interesting_categoryparts_ordinal) == 0:
self._interesting_categoryparts_level = 0
self.log("interesting_categoryparts_ordinal = %s" % self._interesting_categoryparts_ordinal)
tssid = '#%s' % self.seriessetid() if self.seriessetid() else ''
resource_usage.report('%s init %s %s ts %s coeffs' % (type(self).__name__, tssid, resource_timeseries, resource_coefficients))
class GalaxyLayoutThread(QtCore.QThread):
def __init__(self, root):
QtCore.QThread.__init__(self)
self.root = root
self.phases = [
# These don't change no matter what
self.task_preamble,
self.task_computeCoefficients,
self.task_computeOptimalDistances,
# These depend on what the central node is
self.task_cutoffOuterLinks,
self.task_cutoffCentralLinks,
self.task_makeLinks,
self.task_tupliseNodeLinks,
self.task_removeUnattachedNodes,
self.task_findRangeOfCentralLinks,
self.task_computeSecondaryNodeIndexList,
self.task_computeAlphae,
self.task_computeOptimalDistancesToCenterNode,
self.task_placeNodes,
self.task_setupAdjust,
self.task_setupAdjustAllNodes,
self.task_adjustAllNodes,
self.task_noLongerWorking,
self.task_end,
]
self.phase = self.phases[0]
self.phaseIndex = 0
self.displayPhaseSignal = dict()
self.phasesByName = dict()
self.maxIndex = -1
self.entryByStep = False
for _index, _phase in enumerate(self.phases):
self.displayPhaseSignal[_phase.__name__] = QtCore.Signal()
QtCore.QObject.connect(
self, QtCore.SIGNAL(_phase.__name__ + "Signal ()"), _phase, QtCore.Qt.QueuedConnection
)
self.phasesByName[_phase.__name__] = _index
step = QtCore.Signal()
QtCore.QObject.connect(self, QtCore.SIGNAL("step ()"), self.doPhase, QtCore.Qt.QueuedConnection)
displayReset = QtCore.Signal() # Reset - call when new database is loaded
QtCore.QObject.connect(self, QtCore.SIGNAL("displayReset ()"), self.reset, QtCore.Qt.QueuedConnection)
self.quit = QtCore.Signal()
QtCore.QObject.connect(self, QtCore.SIGNAL("quit ()"), self.doQuit)
self.START = time.time()
self.suspend = True
def task_preamble(self):
if not self.enterThisFunction():
return
self.resource_usage = ConciseMonitor()
self.root.galaxy.view.links = []
self.root.galaxy.view.visibleNodeDetails.setNodeList([self.root.centerNode()])
try:
self.root.nodes[self.root.centerNode()].hiding = False
except:
self.root.setCenterNode(0)
self.root.nodes[0].hiding = False
self.nextStep()
def showSparse(self, matrix, integer=False):
result = "["
for i in matrix:
result += "{"
added = False
for j in i:
if integer:
result += "%d:%4d" % (j, i[j]) + ", "
else:
result += "%d:%.2f" % (j, i[j]) + ", "
added = True
if added:
result = result[:-2]
result += "}"
result += "]"
return result
def task_computeCoefficients(self):
if not self.enterThisFunction():
return
self.computeListOfVisibleNodes()
self.maxCoefficient = self.root.cm.maxabscoeff()
self.bestCoefficients = self.root.cm.best
self.secondaryCoefficients = self.root.cm.secondary
self.directions = self.root.cm.direction
self.absNormalizedCoefficients = self.root.cm.norm
# Create an empty vector that we'll use later
self.optimalCentralLinkLength = self.makeEmptyRowVector()
self.nextStep()
def task_computeOptimalDistances(self):
if not self.enterThisFunction():
return
self.computeListOfVisibleNodes()
self.optimalCrosslinkLength = []
for i in range(self.root.N):
_optimalCrosslinkLength = dict()
_normalizedCoefficientVector = self.absNormalizedCoefficients[i]
for j in _normalizedCoefficientVector:
_distance = int((1.0 - _normalizedCoefficientVector[j]) * self.root.MAX_CROSSLINK_LENGTH)
if _distance < self.root.MINIMUM_SEPARATION:
_distance = self.root.MINIMUM_SEPARATION
_optimalCrosslinkLength[j] = _distance
self.optimalCrosslinkLength.append(_optimalCrosslinkLength)
self.nextStep()
def task_cutoffOuterLinks(self):
if not self.enterThisFunction():
return
self.root.galaxy.view.outerLinkCutoffSlider.slider.value()
self.root.displayedLastTime = self.root.galaxy.view.visibleNodeDetails.nodeList()
_sliderValue = (255 - self.root.galaxy.view.outerLinkCutoffSlider.slider.value()) / 255.0
self.computeListOfVisibleNodes()
_centerNode = self.root.centerNode() ## Value is 0
_newVisibleNodeIndexList = []
_sparse = self.root.cm.sparse ### value is true
_normalizedCoefficients = self.absNormalizedCoefficients[_centerNode]
for i in self.visibleNodeIndexList:
if i == _centerNode:
_newVisibleNodeIndexList.append(_centerNode)
else:
_normalized = 0 if _sparse and i not in _normalizedCoefficients else _normalizedCoefficients[i]
if _normalized >= _sliderValue:
_newVisibleNodeIndexList.append(i)
self.visibleNodeIndexList = (
_newVisibleNodeIndexList
) ## Takes input of the point value will be plotted in cercle
# print self.visibleNodeIndexList
self.nextStep()
def task_cutoffCentralLinks(self):
_categoryData = []
parentData = []
if self.root.db:
_series = self.root.tss.getseries
for i in range(self.root.N):
_seriesElement = _series(i)
if i > 0:
try:
_seriesElement.category().split(":")[2]
except:
parentData.append(i)
_categoryData.append([_seriesElement.category(), _seriesElement.label(), i])
if not self.enterThisFunction():
return
self.root.displayedLastTime = self.root.galaxy.view.visibleNodeDetails.nodeList()
_sliderValue = (255 - self.root.galaxy.view.centralLinkCutoffSlider.slider.value()) / 255.0
self.computeListOfVisibleNodes()
_centerNode = self.root.centerNode() ## Value is 0
_newVisibleNodeIndexList = []
_sparse = self.root.cm.sparse ### value is true
_normalizedCoefficients = self.absNormalizedCoefficients[_centerNode]
for i in self.visibleNodeIndexList:
if i == _centerNode:
_newVisibleNodeIndexList.append(_centerNode)
else:
_normalized = 0 if _sparse and i not in _normalizedCoefficients else _normalizedCoefficients[i]
if _normalized >= _sliderValue or i in parentData:
_newVisibleNodeIndexList.append(i)
self.visibleNodeIndexList = (
_newVisibleNodeIndexList
) ## Takes input of the point value will be plotted in cercle
# print self.visibleNodeIndexList
self.nextStep()
def task_makeLinks(self):
if not self.enterThisFunction():
return
_sliderValue = (255 - self.root.galaxy.view.cutoffSlider.slider.value()) / 255.0
_centerNode = self.root.centerNode()
_sparse = self.root.cm.sparse
for i in self.visibleNodeIndexList:
_node = self.root.nodes[i].galaxy
_node.linksOut = []
_node.linksIn = []
_normalizedCoefficients = self.absNormalizedCoefficients[i]
_bestCoefficients = self.bestCoefficients[i]
_secondaryCoefficients = self.secondaryCoefficients[i]
_directions = self.directions[i]
_maxCoefficient = 1.0 if self.root.cm.maxabscoeff() == 0.0 else self.root.cm.maxabscoeff()
for j in self.visibleNodeIndexList:
if j < i:
_normalized = 0 if _sparse and j not in _normalizedCoefficients else _normalizedCoefficients[j]
if (_normalized >= _sliderValue) or (i == _centerNode) or (j == _centerNode):
_best = 0 if _sparse and j not in _bestCoefficients else _bestCoefficients[j]
if _best != 0:
_secondary = 0 if _sparse and j not in _secondaryCoefficients else _secondaryCoefficients[j]
_direction = 0 if _sparse and j not in _directions else _directions[j]
if _direction == 2:
_link = universe.GalaxyLink(self.root, j, i, _best, _secondary, _maxCoefficient)
else:
_link = universe.GalaxyLink(self.root, i, j, _best, _secondary, _maxCoefficient)
self.nextStep()
def task_tupliseNodeLinks(self):
if not self.enterThisFunction():
return
_allLinks = []
for _nodeIndex in self.visibleNodeIndexList:
_node = self.root.nodes[_nodeIndex].galaxy
for _link in _node.linksIn:
_allLinks.append(_link)
_node.linksIn = tuple(_node.linksIn)
_node.linksOut = tuple(_node.linksOut)
self.root.galaxy.view.links = tuple(_allLinks)
self.nextStep()
def task_removeUnattachedNodes(self):
if not self.enterThisFunction():
return
_centerNode = self.root.centerNode()
_cutoffNonZero = self.root.galaxy.view.centralLinkCutoffSlider.slider.value() != 255
for _index in self.visibleNodeIndexList[:]:
_node = self.root.nodes[_index]
_galaxy = _node.galaxy
if _index == _centerNode:
for _link in _galaxy.linksIn:
_link.zValue = 4
for _link in _galaxy.linksOut:
_link.zValue = 4
else:
if _cutoffNonZero:
if len(_galaxy.linksOut):
None
elif len(_galaxy.linksIn):
None
else:
self.visibleNodeIndexList.remove(_index)
self.nextStep()
def task_removeUnattachedNodes(self):
if not self.enterThisFunction():
return
_centerNode = self.root.centerNode()
_cutoffNonZero = self.root.galaxy.view.outerLinkCutoffSlider.slider.value() != 255
for _index in self.visibleNodeIndexList[:]:
_node = self.root.nodes[_index]
_galaxy = _node.galaxy
if _index == _centerNode:
for _link in _galaxy.linksIn:
_link.zValue = 4
for _link in _galaxy.linksOut:
_link.zValue = 4
else:
if _cutoffNonZero:
if len(_galaxy.linksOut):
None
elif len(_galaxy.linksIn):
None
else:
self.visibleNodeIndexList.remove(_index)
self.nextStep()
def task_findRangeOfCentralLinks(self):
coffList = []
if not self.enterThisFunction():
return
_minCoeff = 1.0e99
_maxCoeff = 0.0
_centerNode = self.root.centerNode()
_bestCoefficients = self.bestCoefficients[_centerNode]
_sparse = self.root.cm.sparse
newCoeff = 1
for _nodeIndex in self.visibleNodeIndexList:
if _nodeIndex != _centerNode:
_coeff = 0 if _sparse and _nodeIndex not in _bestCoefficients else abs(_bestCoefficients[_nodeIndex])
_maxCoeff = max(_maxCoeff, _coeff)
_minCoeff = min(_minCoeff, _coeff)
coffList.append(_coeff)
# print _coeff
# print _nodeIndex
if _coeff < newCoeff:
newCoeff = _coeff
outerNode = _nodeIndex
## much anticipated coefficient
# Calculate the scaling values
# print 'min',newCoeff," node",outerNode
sortedCoffList = sorted(coffList)
# print sortedCoffList
# print sortedCoffList
self.coeffShift = _minCoeff
try:
self.coeffMultiplier = 1.0 / (_maxCoeff - _minCoeff)
except:
self.coeffMultiplier = 0.0
self.coeffShift = 0.0
self.secondaryNodeIndexList = self.visibleNodeIndexList[:]
# Remove the central node from the list of secondary nodes
if _centerNode in self.secondaryNodeIndexList:
self.secondaryNodeIndexList.remove(_centerNode)
if _centerNode not in self.visibleNodeIndexList:
print "The central node appears to be unapparent"
self.nextStep()
def task_computeSecondaryNodeIndexList(self):
if not self.enterThisFunction():
return
_list = self.visibleNodeIndexList[:]
_filteredList = []
# ANDY 2011-11-04 Added this line
self.secondaryNodeIndexList = self.visibleNodeIndexList[:]
"""
for _nodeIndex in _list:
_node = self.root.nodes [_nodeIndex]
if (len (_node.galaxy.linksIn) + len (_node.galaxy.linksOut)) > 0:
_filteredList.append (_nodeIndex)
self.secondaryNodeIndexList = _filteredList
"""
self.nextStep()
def task_computeAlphae(self):
if not self.enterThisFunction():
return
self.alpha = dict() # Dictionary of bearings
if self.root.centerNode() in self.secondaryNodeIndexList:
self.secondaryNodeIndexList.remove(self.root.centerNode())
if len(self.secondaryNodeIndexList):
for _nodeIndex in self.visibleNodeIndexList:
self.alpha[_nodeIndex] = self.Talpha(self.root.centerNode(), self.secondaryNodeIndexList[0], _nodeIndex)
self.nextStep()
def task_computeOptimalDistancesToCenterNode(self):
#### Here we are calculating the length which value should be plotted where
if not self.enterThisFunction():
return
self.root.galaxy.view.visibleNodeDetails.setNodeList(self.visibleNodeIndexList)
_power = (1.0 + (self.root.galaxy.view.centrifugeSlider.slider.value())) / 256.0
_centerNode = self.root.centerNode()
_bestCoefficients = self.bestCoefficients[_centerNode]
_sparse = self.root.cm.sparse
for _nodeIndex in self.visibleNodeIndexList:
if _nodeIndex != _centerNode:
_best = 0 if _sparse and _nodeIndex not in _bestCoefficients else abs(_bestCoefficients[_nodeIndex])
_coeff = (_best - self.coeffShift) * self.coeffMultiplier
_centrifugedCoefficient = 1.0 - math.pow(_coeff, _power)
self.optimalCentralLinkLength[_nodeIndex] = (
_centrifugedCoefficient * self.root.SCALING + self.root.OFFSET
)
# print self.optimalCentralLinkLength [_nodeIndex]
length = self.optimalCentralLinkLength[_nodeIndex]
# print length,"node ",_nodeIndex
self.nextStep()
def computeListOfVisibleNodes(self):
self.visibleNodeIndexList = []
for _nodeIndex, _node in enumerate(self.root.nodes):
if not _node.hiding:
self.visibleNodeIndexList.append(_nodeIndex)
if not self.root.centerNode() in self.visibleNodeIndexList:
self.visibleNodeIndexList.append(self.root.centerNode())
def task_placeNodes(self):
if not self.enterThisFunction():
return
# Reset the node position information
self.root.galaxy.view.visibleNodeDetails.resetPositions()
# Place the primary node
self.root.galaxy.view.visibleNodeDetails.setPosition(self.root.centerNode(), QtCore.QPointF(0, 0))
# Place non-central nodes
if len(self.secondaryNodeIndexList):
self.secondaryNode = self.secondaryNodeIndexList[0]
for _nodeIndex in self.secondaryNodeIndexList:
_currentPosition = self.root.nodes[_nodeIndex].galaxy.previousEndpoint
try:
self.alpha[_nodeIndex] = math.atan(_currentPosition.y() / _currentPosition.x())
if _currentPosition.x() < 0.0:
self.alpha[_nodeIndex] -= math.pi
except:
self.alpha[_nodeIndex] = random.uniform(
0.0, 2.0 * math.pi
) # Start with nodes with random alphae so when crosslink cutoff is max we have some spread
_intensity = self.distance(_nodeIndex)
_alpha = self.alpha[_nodeIndex]
self.root.galaxy.view.visibleNodeDetails.setPosition(
_nodeIndex, QtCore.QPointF(_intensity * math.cos(_alpha), _intensity * math.sin(_alpha))
)
# print self.visibleNodeIndexList
self.nextStep()
def task_setupAdjust(self):
if not self.enterThisFunction():
return
self.DEGREES = math.pi / 180.0
# Usual values
self.alphaDelta = 130.0 * self.DEGREES
self.alphaDeltaReductionFactor = 0.7
self.alphaDeltaSmallest = 5.0 * self.DEGREES
self.nextStep()
def task_setupAdjustAllNodes(self):
# See how many iterations we need at most - this is just so we can indicate progress
# self.iterations = int (math.log10 (self.alphaDeltaSmallest / self.alphaDelta) / math.log10 (self.alphaDeltaReductionFactor)) + 1
self.iteration = 0
self.thisNode = 1 # This has to start at 1, not zero. We ignore the first value in the list
self.alphaDelta = 130.0 * self.DEGREES
if len(self.secondaryNodeIndexList):
self.lastNode = self.secondaryNodeIndexList[-1]
self.changesThisIteration = False
self.nextStep()
else:
# Skip the next step
self.phase = self.phases[self.phases.index(self.phase) + 1]
self.nextStep()
def task_adjustAllNodes(self):
if not self.enterThisFunction():
return # Don't want to reset timer
_startTime = time.time()
self.DECLUTTER_THRESHOLD = 400 # This is the square of the repulsion distance
self.declutterSliderValue = self.root.galaxy.view.declutterSlider.slider.value()
self.lsZeroCache = []
_secondaryNodeIndexList = self.secondaryNodeIndexList
_alphaDeltaReductionFactor = self.alphaDeltaReductionFactor
self.lsZeroCache = [-1] * self.root.N
if len(_secondaryNodeIndexList) > 1:
_thisNode = _secondaryNodeIndexList[self.thisNode]
while (time.time() - _startTime) < 0.01:
self.changesMade = self.lsIterationOnOneNode(_thisNode, _secondaryNodeIndexList, self.alphaDelta)
self.changesThisIteration = True
if self.alphaDelta < self.alphaDeltaSmallest:
self.nextStep()
return
else:
if _thisNode == self.lastNode:
if self.changesThisIteration:
self.thisNode = 1
_thisNode = _secondaryNodeIndexList[self.thisNode]
self.alphaDelta *= _alphaDeltaReductionFactor
self.iteration += 1
self.changesThisIteration = False
else:
self.nextStep()
return
else:
self.thisNode += 1
_thisNode = _secondaryNodeIndexList[self.thisNode]
self.repeatStep()
else:
self.nextStep()
def task_noLongerWorking(self):
if not self.enterThisFunction():
return
self.nextStep()
def task_end(self):
if not self.enterThisFunction():
return
def makeEmptyRowVector(self):
_vector = []
for i in range(self.root.N):
_vector.append(0)
return _vector
def makeEmptyTriangularMatrix(self):
_matrix = []
for i in range(self.root.N):
_row = []
for j in range(i):
_row.append(0)
_matrix.append(_row)
return _matrix
def distance(self, j):
return self.optimalCentralLinkLength[j]
def suspendRedraw(self, state):
self.suspend = state
def getOptimalCrosslinkLength(self, nodeIndex1, nodeIndex2):
_lengths = self.optimalCrosslinkLength
if nodeIndex2 in _lengths[nodeIndex1]:
return _lengths[nodeIndex1][nodeIndex2]
else:
return self.root.MAX_CROSSLINK_LENGTH
def Talpha(self, centralNodeIndex, nodeIndex1, nodeIndex2):
if nodeIndex1 == nodeIndex2:
return 3.14
elif nodeIndex1 == centralNodeIndex:
return 3.14
elif nodeIndex2 == centralNodeIndex:
return 3.14
_a = self.distance(nodeIndex1)
_b = self.distance(nodeIndex2)
_d = self.root.galaxyLayoutThread.getOptimalCrosslinkLength(nodeIndex1, nodeIndex2)
_a2 = _a * _a
_b2 = _b * _b
_d2 = _d * _d
try:
result = math.acos((_a2 + _b2 - _d2) / (2.0 * _a * _b))
except:
result = 3.14
return result
def placeNode(self, centralNodeIndex, thisNodeIndex):
_optimalDistance = self.distance(thisNodeIndex)
_x = _optimalDistance * math.cos(self.alpha[thisNodeIndex])
_y = _optimalDistance * math.sin(self.alpha[thisNodeIndex])
self.root.galaxy.view.visibleNodeDetails.setPosition(thisNodeIndex, QtCore.QPointF(_x, _y))
def sumOfSquares(self, centralNodeIndex, thisNode, _listOfVisibleNodes):
sum = 0
_nodeIterationList = _listOfVisibleNodes[:]
if thisNode in _nodeIterationList:
_nodeIterationList.remove(thisNode)
posThis = self.root.galaxy.view.visibleNodeDetails.position(thisNode)
posThisX = posThis.x()
posThisY = posThis.y()
_sliderValue = (255 - self.root.galaxy.view.cutoffSlider.slider.value()) / 255.0
_normalizedCoefficients = self.absNormalizedCoefficients[thisNode]
_sparse = self.root.cm.sparse
for nodeIndex in _nodeIterationList:
_normalized = (
0 if _sparse and nodeIndex not in _normalizedCoefficients else _normalizedCoefficients[nodeIndex]
)
if _normalized < _sliderValue:
continue
posIndex = self.root.galaxy.view.visibleNodeDetails.position(nodeIndex)
distanceX = posThisX - posIndex.x()
distanceY = posThisY - posIndex.y()
# Make squares by multiplying numbers by themselves. Don't use math.pow: it's way too slow
effect = distanceX * distanceX + distanceY * distanceY
_term = self.getOptimalCrosslinkLength(thisNode, nodeIndex)
optimal = _term * _term
_diff = abs(effect - optimal)
if effect < self.DECLUTTER_THRESHOLD:
_power = 1.0 + self.declutterSliderValue / 500.0
_addition = math.pow(
_diff
* (
1
+ (
(self.declutterSliderValue / 255.0)
* (self.DECLUTTER_THRESHOLD - effect)
/ self.DECLUTTER_THRESHOLD
)
),
_power,
)
sum += _addition
else:
sum += _diff
return sum
def thisFunction(self):
return inspect.stack()[2][3]
def enterThisFunction(self):
if self.entryByStep:
self.entryByStep = False
else:
self.BIG_START = time.time()
self.root.galaxyTime = ConciseMonitor()
self.START = time.time()
self.root.galaxy.view.lastTimeDisplayed = time.time()
_requestedFunction = self.thisFunction()
_requestedIndex = self.phasesByName[_requestedFunction]
_currentIndex = self.phasesByName[self.phase.__name__]
if _currentIndex > self.maxIndex:
self.maxIndex = _currentIndex
if _requestedIndex > self.maxIndex:
return False
else:
self.phase = self.phases[_requestedIndex]
return True
def makeListOfVisibleNodes(self):
if not self.enterThisFunction():
return
self.computeListOfVisibleNodes()
self.nextStep()
def nextStep(self):
self.resource_usage.report("nextStep: %s" % (self.phase.__name__))
self.resource_usage = ConciseMonitor()
# Report how long the last phase took
##print self.phase.__name__
if self.phase.__name__ == "task_preamble":
self.START = time.time()
self.BIG_START = self.START
self.START = self.root.INSTRUMENT_CENTRIFUGE(self.phase.__name__, self.START)
##print self.START
self.iteration = 0
# prepare for next phase
self.phase = self.phases[self.phases.index(self.phase) + 1]
# Tell the GUI thread to move on
self.root.galaxy.view.display.emit()
def repeatStep(self):
self.root.galaxy.view.display.emit()
def reset(self):
self.phaseIndex = 0
self.phase = self.phases[0]
self.iteration = 0
self.maxIndex = -1
self.entryByStep = False
def doPhase(self):
if not self.suspend:
if self.iteration == 0:
self.START = time.time()
self.entryByStep = True
self.phase()
def doQuit(self):
None
def run(self):
self.setPriority(QtCore.QThread.LowestPriority)
self.exec_() # Wait for events directed to this thread
def lsIterationOnOneNode(self, nodeIndex, nodeIndexList, alphaDelta):
if self.lsZeroCache[nodeIndex] == -1:
lsZero = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList)
else:
lsZero = self.lsZeroCache[nodeIndex]
_initialAlpha = self.alpha[nodeIndex]
self.alpha[nodeIndex] += alphaDelta
self.placeNode(self.root.centerNode(), nodeIndex)
lsPlus = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList)
if lsPlus >= lsZero:
self.alpha[nodeIndex] = _initialAlpha - alphaDelta
self.placeNode(self.root.centerNode(), nodeIndex)
lsMinus = self.sumOfSquares(self.root.centerNode(), nodeIndex, nodeIndexList)
if lsMinus >= lsZero:
self.alpha[nodeIndex] = _initialAlpha
self.placeNode(self.root.centerNode(), nodeIndex)
return False
else:
self.lsZeroCache[nodeIndex] = lsMinus
else:
self.lsZeroCache[nodeIndex] = lsPlus
return True
def time_one(db, tssid):
keep = list() # keep is required to prevent python GC from deleting unused tss and coeff matrices
for usesparse in (False,True):
resuse = ConciseMonitor()
tss = TimeSeriesSet(db,tssid) # ,usesparse)
tss._usesparse = usesparse
keep.append(tss)
cindex = tss.coeffindex() # dict of { coeffid, description } for all coefficient matrices
keep.append(tss.coefficients())
for coeffid in sorted(cindex.keys()):
localuse = ConciseMonitor()
cm = tss.loadcoefficients(db,coeffid)
keep.append(cm)
localuse.report('load %10.10s %30.30s cardinality=%s' % (type(cm).__name__, cm.description(), cm.cardinality()))
sumuse = ConciseMonitor()
coeffs, n = cm.readyloop()
sum = 0
if cm.sparse:
for i in range(n):
row = coeffs[i]
for j in row:
sum += row[j]
else:
for i in range(n):
row = coeffs[i]
for j in range(n):
sum += row[j]
sumuse = sumuse.report('sum of all %s cells = %s explicit' % (n*n, sum))
if cm.sparse:
coeffs, n = cm.readyloop()
sum = 0
for i in range(n):
for j in coeffs[i]:
sum += coeffs.get(i,j)
sumuse = sumuse.report('sum of all %s cells = %s using Sparse.get(i,j)' % (n*n, sum))
sumuse = sumuse.report('sum of all %s cells = %s implicit by row' % (n*n, cm.sum_by_rows()))
sumuse.report('sum of all %s cells = %s implicit by col' % (n*n, cm.sum_by_cols()))
resuse.report('test_one %s ^^^^^^^^^^^^^^^^^' % cm.matrix_type())
def time_nested_loop(n):
loopuse = ConciseMonitor()
for i in range(n):
for j in range(n):
pass
loopuse.report('loopuse - empty nested loop of %s x %s = %s iterations' % (n,n,n*n))
