def saveStats(args): i, theta, startDate, endDate, recordStep = args resultsFileName = outputDir + "SimStats" + str(i) + ".pkl" try: with open(resultsFileName) as f: pass except IOError as e: featureInds= numpy.ones(targetGraph.vlist.getNumFeatures(), numpy.bool) featureInds[HIVVertices.dobIndex] = False featureInds[HIVVertices.infectionTimeIndex] = False featureInds[HIVVertices.hiddenDegreeIndex] = False featureInds[HIVVertices.stateIndex] = False featureInds = numpy.arange(featureInds.shape[0])[featureInds] matcher = GraphMatch("PATH", alpha=0.5, featureInds=featureInds, useWeightM=False) graphMetrics = HIVGraphMetrics2(targetGraph, 1.0, matcher, float(endDate)) times, infectedIndices, removedIndices, graph = HIVModelUtils.simulate(thetaArray[i], startDate, endDate, recordStep, M, graphMetrics) times, vertexArray, removedGraphStats = HIVModelUtils.generateStatistics(graph, startDate, endDate, recordStep) stats = times, vertexArray, removedGraphStats, graphMetrics.dists, graphMetrics.graphDists, graphMetrics.labelDists Util.savePickle(stats, resultsFileName)
def profileSimulate(self): startDate, endDate, recordStep, printStep, M, targetGraph = HIVModelUtils.realSimulationParams() meanTheta, sigmaTheta = HIVModelUtils.estimatedRealTheta() meanTheta = numpy.array([337, 1.4319, 0.211, 0.0048, 0.0032, 0.5229, 0.042, 0.0281, 0.0076, 0.0293]) undirected = True graph = HIVGraph(M, undirected) logging.info("Created graph: " + str(graph)) alpha = 2 zeroVal = 0.9 p = Util.powerLawProbs(alpha, zeroVal) hiddenDegSeq = Util.randomChoice(p, graph.getNumVertices()) rates = HIVRates(graph, hiddenDegSeq) model = HIVEpidemicModel(graph, rates) model.setT0(startDate) model.setT(startDate+100) model.setRecordStep(recordStep) model.setPrintStep(printStep) model.setParams(meanTheta) logging.debug("MeanTheta=" + str(meanTheta)) ProfileUtils.profile('model.simulate()', globals(), locals())
def findDerivative(args): pertScale, startDate, endDate, recordStep, M, targetGraph, seed = args numpy.random.seed(seed) meanTheta, sigmaTheta = HIVModelUtils.toyTheta() epsilon = 5.0 undirected = True alpha = 2 zeroVal = 0.9 p = Util.powerLawProbs(alpha, zeroVal) graph = HIVGraph(M, undirected) featureInds= numpy.ones(graph.vlist.getNumFeatures(), numpy.bool) featureInds[HIVVertices.dobIndex] = False featureInds[HIVVertices.infectionTimeIndex] = False featureInds[HIVVertices.hiddenDegreeIndex] = False featureInds[HIVVertices.stateIndex] = False featureInds = numpy.arange(featureInds.shape[0])[featureInds] matcher = GraphMatch("PATH", alpha=0.5, featureInds=featureInds, useWeightM=False) abcParams = HIVABCParameters(meanTheta, sigmaTheta, pertScale) newTheta = abcParams.perturbationKernel(meanTheta) undirected = True graph = HIVGraph(M, undirected) graphMetrics = HIVGraphMetrics2(targetGraph, epsilon, matcher, float(endDate)) graphMetrics.breakDist = 1.0 hiddenDegSeq = Util.randomChoice(p, graph.getNumVertices()) rates = HIVRates(graph, hiddenDegSeq) model = HIVEpidemicModel(graph, rates, T=float(endDate), T0=float(startDate), metrics=graphMetrics) model.setRecordStep(recordStep) model.setParams(meanTheta) times, infectedIndices, removedIndices, graph = model.simulate(True) return abs(0.7 - graphMetrics.distance())/numpy.linalg.norm(newTheta-meanTheta)
from apgl.predictors.ABCSMC import ABCSMC import logging import sys import numpy import multiprocessing assert False, "Must run with -O flag" FORMAT = "%(levelname)s:root:%(process)d:%(message)s" logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=FORMAT) numpy.set_printoptions(suppress=True, precision=4, linewidth=150) numpy.seterr(invalid='raise') resultsDir = PathDefaults.getOutputDir() + "viroscopy/toy/" startDate, endDate, recordStep, M, targetGraph = HIVModelUtils.toySimulationParams() epsilonArray = numpy.ones(10)*0.4 logging.debug("Total time of simulation is " + str(endDate-startDate)) posteriorSampleSize = 30 breakDist = 0.5 logging.debug("Posterior sample size " + str(posteriorSampleSize)) def createModel(t): """ The parameter t is the particle index. """ undirected = True graph = HIVGraph(M, undirected) alpha = 2
from exp.viroscopy.model.HIVModelUtils import HIVModelUtils import matplotlib.pyplot as plt """ This is the epidemic model for the HIV spread in cuba. We try to get more bisexual contacts """ assert False, "Must run with -O flag" logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) numpy.seterr(all='raise') numpy.random.seed(24) numpy.set_printoptions(suppress=True, precision=4, linewidth=100) startDate, endDate, recordStep, M, targetGraph = HIVModelUtils.realSimulationParams() endDate = startDate + 10000 meanTheta, sigmaTheta = HIVModelUtils.estimatedRealTheta() meanTheta = numpy.array([ 50, 0.5131, 0.3242, 0.1, 0.0001, 0.0, 325, 0.34, 0.001, 0.1, 0.1, 0.1]) outputDir = PathDefaults.getOutputDir() + "viroscopy/" undirected = True graph = HIVGraph(M, undirected) logging.info("Created graph: " + str(graph)) alpha = 2 zeroVal = 0.9 p = Util.powerLawProbs(alpha, zeroVal) hiddenDegSeq = Util.randomChoice(p, graph.getNumVertices()) rates = HIVRates(graph, hiddenDegSeq)
from exp.viroscopy.model.HIVEpidemicModel import HIVEpidemicModel from exp.viroscopy.model.HIVRates import HIVRates from exp.viroscopy.model.HIVModelUtils import HIVModelUtils """ This is the epidemic model for the HIV spread in cuba. We repeat the simulation a number of times and average the results. The purpose is to test the ABC model selection by using a known value of theta. """ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) numpy.seterr(all='raise') numpy.random.seed(24) numpy.set_printoptions(suppress=True, precision=4, linewidth=100) startDate, endDate, recordStep, M = HIVModelUtils.toySimulationParams(False) endDate += HIVModelUtils.toyTestPeriod numRepetitions = 1 undirected = True outputDir = PathDefaults.getOutputDir() + "viroscopy/toy/" theta, sigmaTheta = HIVModelUtils.toyTheta() graphList = [] for j in range(numRepetitions): graph = HIVGraph(M, undirected) logging.debug("Created graph: " + str(graph)) alpha = 2
from apgl.predictors.ABCSMC import ABCSMC import logging import sys import numpy import multiprocessing assert False, "Must run with -O flag" FORMAT = "%(levelname)s:root:%(process)d:%(message)s" logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=FORMAT) numpy.set_printoptions(suppress=True, precision=4, linewidth=150) numpy.seterr(invalid='raise') resultsDir = PathDefaults.getOutputDir() + "viroscopy/real/" startDate, endDates, numRecordSteps, M, targetGraph = HIVModelUtils.realSimulationParams() if len(sys.argv) > 1: numProcesses = int(sys.argv[1]) else: numProcesses = multiprocessing.cpu_count() posteriorSampleSize = 20 breakDist = 0.4 logging.debug("Posterior sample size " + str(posteriorSampleSize)) for i, endDate in enumerate(endDates): logging.debug("="*10 + "Starting new simulation batch" + "="*10) logging.debug("Total time of simulation is " + str(endDate-startDate)) epsilonArray = numpy.ones(15)*0.4
def testSimulate2(self): startDate = 0.0 endDate = 100.0 M = 1000 meanTheta, sigmaTheta = HIVModelUtils.estimatedRealTheta() undirected = True graph = HIVGraph(M, undirected) alpha = 2 zeroVal = 0.9 p = Util.powerLawProbs(alpha, zeroVal) hiddenDegSeq = Util.randomChoice(p, graph.getNumVertices()) meanTheta[4] = 0.1 recordStep = 10 printStep = 10 rates = HIVRates(graph, hiddenDegSeq) model = HIVEpidemicModel(graph, rates, endDate, startDate) model.setRecordStep(recordStep) model.setPrintStep(printStep) model.setParams(meanTheta) initialInfected = graph.getInfectedSet() times, infectedIndices, removedIndices, graph = model.simulate(True) #Now test the final graph edges = graph.getAllEdges() for i, j in edges: if graph.vlist.V[i, HIVVertices.genderIndex] == graph.vlist.V[j, HIVVertices.genderIndex] and (graph.vlist.V[i, HIVVertices.orientationIndex] != HIVVertices.bi or graph.vlist.V[j, HIVVertices.orientationIndex] != HIVVertices.bi): self.fail() finalInfected = graph.getInfectedSet() finalRemoved = graph.getRemovedSet() self.assertEquals(numpy.intersect1d(initialInfected, finalRemoved).shape[0], len(initialInfected)) #Test case where there is no contact meanTheta = numpy.array([100, 0.95, 1, 1, 0, 0, 0, 0, 0, 0, 0], numpy.float) times, infectedIndices, removedIndices, graph, model = runModel(meanTheta) self.assertEquals(len(graph.getInfectedSet()), 100) self.assertEquals(len(graph.getRemovedSet()), 0) self.assertEquals(graph.getNumEdges(), 0) heteroContactRate = 0.1 meanTheta = numpy.array([100, 0.95, 1, 1, 0, 0, heteroContactRate, 0, 0, 0, 0], numpy.float) times, infectedIndices, removedIndices, graph, model = runModel(meanTheta) self.assertEquals(len(graph.getInfectedSet()), 100) self.assertEquals(len(graph.getRemovedSet()), 0) edges = graph.getAllEdges() for i, j in edges: self.assertNotEqual(graph.vlist.V[i, HIVVertices.genderIndex], graph.vlist.V[j, HIVVertices.genderIndex]) #Number of conacts = rate*people*time infectedSet = graph.getInfectedSet() numHetero = (graph.vlist.V[list(infectedSet), HIVVertices.orientationIndex] == HIVVertices.hetero).sum() self.assertTrue(abs(numHetero*endDate*heteroContactRate- model.getNumContacts()) < 100) heteroContactRate = 0.01 meanTheta = numpy.array([100, 0.95, 1, 1, 0, 0, heteroContactRate, 0, 0, 0, 0], numpy.float) times, infectedIndices, removedIndices, graph, model = runModel(meanTheta) infectedSet = graph.getInfectedSet() numHetero = (graph.vlist.V[list(infectedSet), HIVVertices.orientationIndex] == HIVVertices.hetero).sum() self.assertAlmostEqual(numHetero*endDate*heteroContactRate/100, model.getNumContacts()/100.0, 0)
import matplotlib.pyplot as plt from apgl.graph.GraphStatistics import GraphStatistics from apgl.util.PathDefaults import PathDefaults from apgl.util.Util import Util from apgl.util.Latex import Latex from apgl.predictors.ABCSMC import loadThetaArray from exp.viroscopy.model.HIVModelUtils import HIVModelUtils from exp.viroscopy.model.HIVVertices import HIVVertices from exp.sandbox.GraphMatch import GraphMatch from exp.viroscopy.model.HIVGraphMetrics2 import HIVGraphMetrics2 assert False, "Must run with -O flag" logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) numpy.set_printoptions(suppress=True, precision=4, linewidth=150) startDate, endDates, numRecordSteps, M, targetGraph = HIVModelUtils.realSimulationParams() plotStyles = ['k-', 'kx-', 'k+-', 'k.-', 'k*-'] saveResults = False graphStats = GraphStatistics() N = 8 t = 0 maxT = 20 #We plot some stats for the ideal simulated epidemic #and those epidemics found using ABC. def saveStats(args): i, theta, startDate, endDate, recordStep = args resultsFileName = outputDir + "SimStats" + str(i) + ".pkl"
from apgl.predictors.ABCSMC import loadThetaArray from exp.viroscopy.model.HIVModelUtils import HIVModelUtils from exp.viroscopy.model.HIVVertices import HIVVertices from exp.sandbox.GraphMatch import GraphMatch from exp.viroscopy.model.HIVGraphMetrics2 import HIVGraphMetrics2 assert False, "Must run with -O flag" logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) numpy.set_printoptions(suppress=True, precision=4, linewidth=150) plotStyles = ['k-', 'kx-', 'k+-', 'k.-', 'k*-'] resultsDir = PathDefaults.getOutputDir() + "viroscopy/toy/theta/" outputDir = resultsDir + "stats/" startDate, endDate, recordStep, M, targetGraph = HIVModelUtils.toySimulationParams() endDate += HIVModelUtils.toyTestPeriod saveResults = False graphStats = GraphStatistics() N = 16 t = 0 maxT = 10 realTheta, sigmaTheta = HIVModelUtils.toyTheta() minVal = 10 for i in range(maxT): thetaArray, distArray = loadThetaArray(N, resultsDir, i)
graph = HIVGraph(M, undirected) graphMetrics = HIVGraphMetrics2(targetGraph, epsilon, matcher, float(endDate)) graphMetrics.breakDist = 1.0 hiddenDegSeq = Util.randomChoice(p, graph.getNumVertices()) rates = HIVRates(graph, hiddenDegSeq) model = HIVEpidemicModel(graph, rates, T=float(endDate), T0=float(startDate), metrics=graphMetrics) model.setRecordStep(recordStep) model.setParams(meanTheta) times, infectedIndices, removedIndices, graph = model.simulate(True) return abs(0.7 - graphMetrics.distance())/numpy.linalg.norm(newTheta-meanTheta) startDate, endDate, recordStep, M, targetGraph = HIVModelUtils.toySimulationParams() numReps = 8 pertScales = [0.01, 0.05, 0.1, 0.2] derivatives = numpy.zeros((numReps, len(pertScales))) for j, pertScale in enumerate(pertScales): logging.debug(pertScale) paramList = [] for i in range(numReps): paramList.append((pertScale, startDate, endDate, recordStep, M, targetGraph, i)) pool = multiprocessing.Pool(multiprocessing.cpu_count()) resultIterator = pool.map(findDerivative, paramList) #resultIterator = map(findDerivative, paramList) pool.terminate()