def runSCG(args): fileName, spcereg, offset, seq, batch, l, readIterFunc = args reader = HairDataReader(fileName, {'type':'anim2'}) mat = [] reader.seek(seq*batch) for i in xrange(batch): res = readIterFunc(reader, i, mat, offset, spcereg) if res is None: logging.warning("Unexpected break at runSCG, frame %d" % i) break reader.close() printMP(l, "Finish read batch %d..." % seq) return seq, mat
def SCGetMatrixAndHeaderMP(fileName, readIterFunc, nFrame=None): reader = HairDataReader(fileName, {'type':'anim2'}) factor = para.factor offset = factor * 3 spcereg = para.lambda_balance # regularize the space item if not nFrame: nFrame = reader.nFrame print "Reading frame %d :" % nFrame pool = Pool() m = Manager() l = m.Lock() batchsz = 10 job_args = [(fileName, spcereg, offset, i, batchsz, l, readIterFunc) for i in xrange(nFrame/batchsz) ] mat = pool.map(runSCG, job_args) pool.close() reader.close() mat.sort(key=lambda x: x[0]) mat2 = [] for item in mat: for arr in item[1]: mat2.append(arr) mat = None X = np.hstack(mat2).transpose() mat2 = None print "\rFinished Reading!" header = HairHeader() header.nParticle = reader.nParticle header.factor = factor header.nHair = header.nParticle / factor assert(X.dtype==np.float64) return X, header, XWrapper(X, offset*2)