def save(self): """Saves ourself to disk""" import cPickle as pickle #from scipy.sparse import lil_matrix as sparsemat from scipy.sparse import csr_matrix as sparsemat #M = M.tocsr() if self.tempflow: return fname = os.path.join(self.g.prefix, 'flows', self.id+'.pickle') try: os.makedirs(os.path.dirname(fname)) except OSError: pass if 0: todump = {} for k in self.scores: todump[k] = sparsemat(self.scores[k]) else: todump = self.scores pickle.dump(todump, open(fname, 'wb'), -1)
def save(self): """Saves ourself to disk""" import cPickle as pickle #from scipy.sparse import lil_matrix as sparsemat from scipy.sparse import csr_matrix as sparsemat #M = M.tocsr() if self.tempflow: return fname = os.path.join(self.g.prefix, 'flows', self.id + '.pickle') try: os.makedirs(os.path.dirname(fname)) except OSError: pass if 0: todump = {} for k in self.scores: todump[k] = sparsemat(self.scores[k]) else: todump = self.scores pickle.dump(todump, open(fname, 'wb'), -1)
def getEdgeMatrix(self, srclayer, dstlayer, srckeys=None, dstkeys=None, usesparse=1): """Computes a matrix of weights that transforms from srclayer to dstlayer. i.e., you have a vector V_s of values from srclayer, and this function returns M_ds. Then you can do V_d = np.dot(M_ds, V_s). Returns (M_ds, list of srclayer keys, list of dstlayer keys). You can optionally pass in lists of srckeys and dstkeys. If so, then only fills in values that exist in these lists. If src and dst layers are the same, then initializes the matrix with identity. Otherwise, initializes the matrix with 0s. If usesparse is true (default), then uses sparse matrices. Notes: - we initialize data using lil_matrix, because it's fastest to modify - we convert to csr_matrix at the end, because that's fastest to multiply """ import scipy as sp import scipy.sparse as sparse from scipy.sparse import lil_matrix as sparsemat times = [time.time()] # init keys and matrix if not srckeys: srckeys = self.nodes(srclayer) if not dstkeys: dstkeys = self.nodes(dstlayer) dstrows = dict((int(self.splitNodekey(dk)[1]), i) for i, dk in enumerate(dstkeys)) times.append(time.time()) ns, nd = len(srckeys), len(dstkeys) assert ns > 0 and nd > 0 if srclayer == dstlayer: if usesparse: M = sparsemat((nd,nd)) M.setdiag(np.ones(nd)) else: M = np.eye(nd) else: if usesparse: M = sparsemat((nd,ns)) else: M = np.zeros((nd, ns)) times.append(time.time()) # fill in the matrix, only if we have something to fill if self.db.hexists('layeredges:%s' % (srclayer), dstlayer): edges = self.getEdges(srckeys, valid=dstlayer, sort=0)[dstlayer] for col, row in enumerate(edges): for nodeid, w in row: if nodeid not in dstrows: continue row = dstrows[nodeid] M[row, col] = w times.append(time.time()) nz = len(M.nonzero()[0]) nels = M.shape[0]*M.shape[1] if nz == 0: M = None else: if ns == nd: # check if it's identity if usesparse: eye = sparsemat((nd,nd)) eye.setdiag(np.ones(nd)) else: eye = np.eye(nd) eye -= M iseye = (len(eye.nonzero()[0]) == 0) if iseye: M = None else: iseye = 0 log(' Matrix from %s (%d) to %s (%d) had %d/%d nonzeros (%0.5f%%) (iseye=%s)' % (srclayer, len(srckeys), dstlayer, len(dstkeys), nz, nels, nz*100.0/float(nels), iseye)) log(' Matrix took: %s' % (getTimeDiffs(times))) if sparse.issparse(M): M = M.tocsr() return (M, srckeys, dstkeys)
def getEdgeMatrix(self, srclayer, dstlayer, srckeys=None, dstkeys=None, usesparse=1): """Computes a matrix of weights that transforms from srclayer to dstlayer. i.e., you have a vector V_s of values from srclayer, and this function returns M_ds. Then you can do V_d = np.dot(M_ds, V_s). Returns (M_ds, list of srclayer keys, list of dstlayer keys). You can optionally pass in lists of srckeys and dstkeys. If so, then only fills in values that exist in these lists. If src and dst layers are the same, then initializes the matrix with identity. Otherwise, initializes the matrix with 0s. If usesparse is true (default), then uses sparse matrices. Notes: - we initialize data using lil_matrix, because it's fastest to modify - we convert to csr_matrix at the end, because that's fastest to multiply """ import scipy as sp import scipy.sparse as sparse from scipy.sparse import lil_matrix as sparsemat times = [time.time()] # init keys and matrix if not srckeys: srckeys = self.nodes(srclayer) if not dstkeys: dstkeys = self.nodes(dstlayer) dstrows = dict( (int(self.splitNodekey(dk)[1]), i) for i, dk in enumerate(dstkeys)) times.append(time.time()) ns, nd = len(srckeys), len(dstkeys) assert ns > 0 and nd > 0 if srclayer == dstlayer: if usesparse: M = sparsemat((nd, nd)) M.setdiag(np.ones(nd)) else: M = np.eye(nd) else: if usesparse: M = sparsemat((nd, ns)) else: M = np.zeros((nd, ns)) times.append(time.time()) # fill in the matrix, only if we have something to fill if self.db.hexists('layeredges:%s' % (srclayer), dstlayer): edges = self.getEdges(srckeys, valid=dstlayer, sort=0)[dstlayer] for col, row in enumerate(edges): for nodeid, w in row: if nodeid not in dstrows: continue row = dstrows[nodeid] M[row, col] = w times.append(time.time()) nz = len(M.nonzero()[0]) nels = M.shape[0] * M.shape[1] if nz == 0: M = None else: if ns == nd: # check if it's identity if usesparse: eye = sparsemat((nd, nd)) eye.setdiag(np.ones(nd)) else: eye = np.eye(nd) eye -= M iseye = (len(eye.nonzero()[0]) == 0) if iseye: M = None else: iseye = 0 log(' Matrix from %s (%d) to %s (%d) had %d/%d nonzeros (%0.5f%%) (iseye=%s)' % (srclayer, len(srckeys), dstlayer, len(dstkeys), nz, nels, nz * 100.0 / float(nels), iseye)) log(' Matrix took: %s' % (getTimeDiffs(times))) if sparse.issparse(M): M = M.tocsr() return (M, srckeys, dstkeys)