def send_recv(x): if neworder.rank() == 0: neworder.send(x, 1) if neworder.rank() == 1: y = neworder.receive(0) neworder.log("MPI: 0 sent {}={} 1 recd {}={}".format( type(x), x, type(y), y)) if y != x: return False return True
def __init__(self, p, n): # states self.s = np.array(range(neworder.size())) # transition matrix self.p = np.identity(neworder.size()) * (1 - neworder.size() * p) + p self.n = n # all begin with unique id and state = rank self.pop = pd.DataFrame({ "id": np.array(range(neworder.rank() * n, neworder.rank() * n + n)), "state": np.full(n, neworder.rank()) })
def greeks(self, pv): neworder.sync() pvs = neworder.gather(pv, 0) if neworder.rank() == 0: neworder.log("PV=%f" % pvs[0]) neworder.log("delta=%f" % ((pvs[1] - pvs[2]) / 2)) neworder.log("gamma=%f" % ((pvs[1] - 2 * pvs[0] + pvs[2]))) neworder.log("vega 10bp=%f" % (pvs[3] - pvs[0]))
def test(self): # generate some movement neworder.transition(self.s, self.p, self.pop, "state") # send migrants for s in range(neworder.size()): if s != neworder.rank(): emigrants = self.pop[self.pop.state == s] #neworder.log("sending %d emigrants to %d" % (len(emigrants), s)) neworder.send(emigrants, s) # remove the emigrants self.pop = self.pop[self.pop.state == neworder.rank()] # receive migrants for s in range(neworder.size()): if s != neworder.rank(): immigrants = neworder.receive(s) #neworder.log("received %d immigrants from %d" % (len(immigrants), s)) self.pop = self.pop.append(immigrants)
def __init__(self, input_files, ht_trans, cache_dir): self.cache_dir = cache_dir # guard for no input data (if more MPI processes than input files) if not len(input_files): raise ValueError("proc {}/{}: no input data".format( no.rank(), no.size())) self.lads = [file.split("_")[1] for file in input_files] # assumes all files in same dir self.data_dir = os.path.dirname(input_files[0]) # store as dict of DFs self.pop = pd.DataFrame() for file in input_files: no.log("reading initial population: %s" % file) data = pd.read_csv(file) data["LAD"] = file.split("_")[1] self.pop = self.pop.append(data) # no.log(self.pop.LC4408_C_AHTHUK11.unique()) # self.cat = self.pop.LC4408_C_AHTHUK11.unique() # "C_AHTHUK11": { # "0": "All categories: Household type", # "1": "One person household", # "2": "Married or same-sex civil partnership couple household", # "3": "Cohabiting couple household", # "4": "Lone parent household", # "5": "Multi-person household" # } self.cat = {"LC4408_C_AHTHUK11": np.array([1, 2, 3, 4, 5])} # NOTE: pandas stores column-major order but numpy view is row major so the matrix looks right but is actually transposed # (no amount of transposing actually changes the memory layout (it just changes the view) # the C++ code assumes the transition matrix is column major (col sums to unity not rows) self.t = pd.read_csv(ht_trans).set_index( "initial state").values / 100.0 # check rows sum to unity assert np.allclose(np.sum(self.t, 1), np.ones(len(self.t))) # TODO get snhp self.snhp = SNHPData.SNHPData(self.cache_dir) self.projection = self.snhp.aggregate(self.lads)
def __init__(self, inputdata, asfr, asmr, asir, asor, ascr, asxr): # guard for no input data (if more MPI processes than input files) if not len(inputdata): raise ValueError("proc {}/{}: no input data".format( neworder.rank(), neworder.size())) self.lads = [file.split("_")[2] for file in inputdata] self.data = pd.DataFrame() for file in inputdata: data = pd.read_csv(file) data["LAD"] = file.split("_")[2] self.data = self.data.append(data) neworder.log("Preprocessing transition data for %s" % ", ".join(self.lads)) self.fertility = ethpop.create_multi(pd.read_csv(asfr), self.lads) self.mortality = ethpop.create_multi(pd.read_csv(asmr), self.lads) self.in_migration = ethpop.local_rates_from_national_rate( ethpop.create_multi(pd.read_csv(asir), self.lads), self.data) self.out_migration = ethpop.create_multi(pd.read_csv(asor), self.lads) self.immigration = ethpop.local_rates_from_absolute( ethpop.create_multi(pd.read_csv(ascr), self.lads), self.data) self.emigration = ethpop.local_rates_from_absolute( ethpop.create_multi(pd.read_csv(asxr), self.lads), self.data) # Force flat rates for testing purposes self.in_migration.Rate = 0.05 self.out_migration.Rate = 0.05 # The actual rates cause exponential growth self.immigration.Rate = 0.01 self.emigration.Rate = 0.005 # use this to identify people (uniquely only within this table) self.counter = len(self.data) # Reformatting of input data is required to match Ethpop categories # actual age is randomised within the bound of the category self.data["Age"] = self.data.DC1117EW_C_AGE - neworder.ustream( len(self.data)) self.data = ethpop.from_census_eth(self.data)
initial_populations = partition(initial_populations, neworder.size()) # running/debug options neworder.log_level = 1 # initialisation neworder.initialisations = { "people": { "module": "population", "class_": "Population", "parameters": [ initial_populations[neworder.rank()], asfr, asmr, asir, asor, ascr, asxr ] } } # define the evolution neworder.timeline = (2011.25, 2050.25, 39) # timestep must be defined in neworder neworder.transitions = { "fertility": "people.births(timestep)", "mortality": "people.deaths(timestep)", "migration": "people.migrations(timestep)", "age": "people.age(timestep)" }
def write_table(self): # TODO define path in config filename = "./examples/people_multi/data/dm_{:.3f}_{}-{}.csv".format( neworder.time, neworder.rank(), neworder.size()) neworder.log("writing %s" % filename) return self.data.to_csv(filename, index=False)
def write_table(self): file = os.path.join( self.data_dir, "dm_{:.3f}_{}-{}.csv".format(no.time, no.rank(), no.size())) no.log("writing final population: %s" % file) self.pop.to_csv(file, index=False)
initial_populations = partition( glob.glob(os.path.join(data_dir, file_pattern)), neworder.size()) # household type transition matrix ht_trans = os.path.join(data_dir, "w_hhtype_dv-tpm.csv") # running/debug options neworder.log_level = 1 # initialisation neworder.initialisations = { "households": { "module": "households", "class_": "Households", "parameters": [initial_populations[neworder.rank()], ht_trans, cache_dir] } } # timestep must be defined in neworder neworder.transitions = {"age": "households.age(timestep)"} # checks to perform after each timestep. Assumed to return a boolean neworder.do_checks = True # Faith # assumed to be methods of class_ returning True if checks pass neworder.checks = {"check": "households.check()"} # Generate output at each checkpoint neworder.checkpoints = {"write_table": "households.write_table()"}
def write_table(self): # TODO define path in config filename = "./examples/world/data/pop2019_{}-{}.csv".format(neworder.rank(), neworder.size()) neworder.log("writing %s" % filename) return self.pop.to_csv(filename, index=False)
def test(): t = test_.Test() if neworder.size() == 1: neworder.log("Skipping MPI tests") return True t.check(send_recv(True)) t.check(send_recv(10)) t.check(send_recv(10.01)) t.check(send_recv("abcdef")) t.check(send_recv([1, 2, 3])) t.check(send_recv({"a": "fghdfkgh"})) x = np.array([1, 4, 9, 16]) if neworder.rank() == 0: neworder.send(x, 1) if neworder.rank() == 1: y = neworder.receive(0) neworder.log("MPI: 0 sent {}={} 1 recd {}={}".format( type(x), x, type(y), y)) t.check(np.array_equal(x, y)) df = pd.read_csv("../../tests/ssm_E09000001_MSOA11_ppp_2011.csv") if neworder.rank() == 0: neworder.log("sending (as csv) df len %d rows from 0" % len(df)) neworder.send_csv(df, 1) if neworder.rank() == 1: dfrec = neworder.receive_csv(0) neworder.log("got (as csv) df len %d rows from 0" % len(dfrec)) t.check(dfrec.equals(df)) if neworder.rank() == 0: neworder.log("sending (pickle) df len %d rows from 0" % len(df)) neworder.send(df, 1) if neworder.rank() == 1: dfrec = neworder.receive(0) neworder.log("got (pickle) df len %d rows from 0" % len(dfrec)) t.check(dfrec.equals(df)) # TODO how to test? neworder.log("process %d syncing..." % neworder.rank()) neworder.sync() neworder.log("process %d synced" % neworder.rank()) i = "rank " + str(neworder.rank()) root = 0 if root == neworder.rank(): neworder.log("broadcasting '%s' from %d" % (i, root)) i = neworder.broadcast(i, root) neworder.log("%d got broadcast: '%s' from %d" % (neworder.rank(), i, root)) t.check(i == "rank 0") # a0 will be different for each proc a0 = np.random.rand(2, 2) if root == neworder.rank(): neworder.log("broadcasting '%s' from %d" % (str(a0), root)) a1 = neworder.broadcast(a0, root) # a1 will equal a0 on rank 0 only neworder.log("%d got broadcast: '%s' from %d" % (neworder.rank(), str(a1), root)) if neworder.rank() == 0: t.check(np.array_equal(a0, a1)) else: t.check(not np.array_equal(a0, a1)) # test ustream/sequence t.check(neworder.indep()) if root == neworder.rank(): u0 = neworder.ustream(1000) u1 = np.zeros(1000) else: u0 = np.zeros(1000) u1 = neworder.ustream(1000) # broadcast u1 from 1 neworder.broadcast(u1, 1) # proc 0 should have 2 different random arrays # proc 1 should have zeros and a random array t.check(not np.array_equal(u0, u1)) # check independent streams u = neworder.ustream(1000) v = neworder.broadcast(u, root) # u == v on broadcasting process only t.check(np.array_equal(u, v) == (neworder.rank() == root)) # test gather x = (neworder.rank() + 1)**2 / 8 a = neworder.gather(x, 0) if neworder.rank() == 0: t.check(np.array_equal(a, [0.125, 0.5])) else: t.check(len(a) == 0) #neworder.log(a) # test scatter if neworder.rank() == 0: a = (np.array(range(neworder.size())) + 1)**2 / 8 else: a = np.zeros(neworder.size()) neworder.log(a) x = neworder.scatter(a, 0) t.check(x == (neworder.rank() + 1)**2 / 8) # test allgather a = np.zeros(neworder.size()) - 1 a[neworder.rank()] = (neworder.rank() + 1)**2 / 8 a = neworder.allgather(a) t.check(np.array_equal(a, np.array([0.125, 0.5]))) # this should probably fail (gather not implemented for int) x = neworder.rank() + 100 a = neworder.gather(x, 0) #neworder.log(type(x)) #neworder.log(type(a)) return not t.any_failed
allcountries = pd.read_csv("./examples/world/data/CountryLookup.csv", encoding='utf-8', sep="\t")["Code"] initial_populations = partition(allcountries, neworder.size()) #initial_populations = [["ALB", "ASM", "ATG"]] # running/debug options neworder.log_level = 1 # initialisation neworder.initialisations = { "people": { "module": "microsynth", "class_": "Microsynth", "parameters": [initial_populations[neworder.rank()]] } } # define the evolution neworder.timeline = (2019, 2030, 11) # timestep must be defined in neworder neworder.transitions = {} # checks to perform after each timestep. Assumed to return a boolean neworder.do_checks = True # Faith # assumed to be methods of class_ returning True if checks pass neworder.checks = {} # Generate output at each checkpoint
def stats(self): neworder.sync() neworder.log("len(pop)=%d" % len(self.pop)) # check we only have status = rank now assert len(self.pop[self.pop.state != neworder.rank()]) == 0