def download_runcard(input_folder, runcard, runname): tar = warmup_name(runcard, runname) gf.print_flush("downloading " + input_folder + "/" + tar) stat = gf.copy_from_grid(input_folder + "/" + tar, tar, args) gf.print_flush("finished copying from grid storage... untarring now") stat += gf.untar_file(tar) return gf.do_shell("rm {0}".format(tar)) + stat
def download_rivet(rivet_folder): tar = os.path.basename(rivet_folder) gf.print_flush("downloading "+rivet_folder) stat = gf.copy_from_grid(rivet_folder, "", args) stat += gf.untar_file(tar) rivet_dir = os.path.basename(os.path.splitext(rivet_folder)[0]) os.environ['RIVET_ANALYSIS_PATH'] = os.getcwd()+"/"+rivet_dir return gf.do_shell("rm {0}".format(tar))+stat
def run_HEJFOG(args): gf.print_flush("TODO HEJFOG not implemented yet") command = "HEJ/bin/HEJFOG" gf.do_shell("chmod +x {0}".format(command)) # TODO: # parse runcard # run HEJ-FOG (with chmod) return 1
def download_runcard(input_folder, runcard, runname): tar = warmup_name(runcard, runname) gf.print_flush("downloading "+input_folder+"/"+tar) stat = gf.copy_from_grid(input_folder+"/"+tar, tar, args) stat += gf.untar_file(tar) # TODO download: # Scale setters return gf.do_shell("rm {0}".format(tar))+stat
def deactivate_environment(args): gf.print_flush("Cleaning up environment...") os.system("source JUNE_env/bin/deactivate") os.system("rm -rf JUNE_env") os.system("rm -rf JUNE") os.system("rm -rf miniconda") if args.world is not "": os.system("rm {}.hdf5".format(args.world)) return None
def download_program(source): tar_name = os.path.basename(source) if not tar_name.endswith("tar.gz"): gf.print_flush("{0} is not a valid path to download".format(source)) return 1 stat = gf.copy_from_grid(source, tar_name, args) stat += gf.untar_file(tar_name) stat += gf.do_shell("rm {0}".format(tar_name)) if gf.DEBUG_LEVEL > 2: gf.do_shell("ls -l") return stat
def bring_files(args): bring_status = 0 if not args.use_cvmfs_lhapdf: gf.print_flush("Using own version of LHAPDF") bring_status += bring_lhapdf(args.lhapdf_grid) bring_status += bring_nnlojet(args.input_folder, args.runcard, args.runname) gf.do_shell("chmod +x {0}".format(args.executable)) if bring_status != 0: gf.print_flush("Not able to bring data from storage. Exiting now.") sys.exit(-95) return bring_status
def setup(): start_time = datetime.datetime.now() gf.print_flush("Start time: {0}".format( start_time.strftime("%d-%m-%Y %H:%M:%S"))) args = gf.parse_arguments() setup_environment(args, args.lhapdf_local) socket_config = None if gf.DEBUG_LEVEL > 1: # Architecture info gf.print_flush("Python version: {0}".format(sys.version)) gf.print_node_info("node_info.log") os.system("lsb_release -a") gf.do_shell("env") gf.do_shell("voms-proxy-info --all") if args.copy_log: # initialise with node name gf.do_shell("hostname >> {0}".format(gf.COPY_LOG)) return args, socket_config
def setup_sockets(args, nnlojet_command, bring_status): host = args.Host port = args.port if bring_status != 0: gf.print_flush("Not able to bring data from gfal, " "removing myself from the pool") socket_sync_str(host, port, handshake="oupsities") sys.exit(-95) gf.print_flush( "Sockets are active, trying to connect to {0}:{1}".format(host, port)) socket_config = socket_sync_str(host, port) if "die" in socket_config: gf.print_flush("Timeout'd by socket server") sys.exit(0) gf.print_flush("Connected to socket server") nnlojet_command += " -port {0} -host {1} {2}".format( port, host, socket_config) return nnlojet_command, socket_config
def run_executable(nnlojet_command): # TODO replace by gf.run_command gf.print_flush(" > Executing command: {0}".format(nnlojet_command)) # Run command status_nnlojet = gf.do_shell(nnlojet_command) if status_nnlojet == 0: gf.print_flush("Command successfully executed") else: gf.print_flush("Something went wrong") gf.DEBUG_LEVEL = 9999 return status_nnlojet
def print_copy_status(args, status_copy): if status_copy == 0: gf.print_flush("Copied over to grid storage!") elif args.Sockets: gf.print_flush( "This was a socketed run so we are copying the grid to stderr just " "in case") gf.do_shell("cat $(ls *.y* | grep -v .txt) 1>&2") status_copy = 0 elif args.Warmup: gf.print_flush("Failure! Outputing vegas warmup to stdout") gf.do_shell("cat $(ls *.y* | grep -v .txt)")
def bring_lhapdf(lhapdf_grid): tmp_tar = "lhapdf.tar.gz" stat = gf.copy_from_grid(lhapdf_grid, tmp_tar, args) gf.print_flush("LHAPDF copy from GRID status: {0}".format(stat)) stat += gf.untar_file(tmp_tar) return gf.do_shell("rm {0}".format(tmp_tar))+stat
def download_latin_hypercube(input_folder, latin_hypercube): lhs = latin_hypercube + ".npy" gf.print_flush("downloading " + input_folder + "/" + lhs) stat = gf.copy_from_grid(input_folder + "/" + lhs, lhs, args) return stat
def download_program(): # TODO read tar and source name from header gf.print_flush("using cvmfs Sherpa") return 0
# ------------------------- Actual run commands ------------------------- def run_sherpa(args): command = "Sherpa RSEED:={0} ANALYSIS_OUTPUT=Sherpa_{0} -f {1}".format( args.seed, config_name(args.runname)) if int(args.events) > 0: command += " -e {0} ".format(args.events) status = gf.run_command(command) return status # ------------------------- MAIN ------------------------- if __name__ == "__main__": if sys.argv[0] and "ENVSET" not in os.environ: gf.print_flush("Setting environment") os.environ["ENVSET"] = "ENVSET" env = "/cvmfs/pheno.egi.eu/HEJ/HEJ_env.sh" os.execvp("bash", ["bash", "-c", "source " + env + " && exec python " + sys.argv[0] + ' "${@}"', "--"] + sys.argv[1:]) start_time = datetime.datetime.now() gf.print_flush("Start time: {0}".format( start_time.strftime("%d-%m-%Y %H:%M:%S"))) args = gf.parse_arguments() lhapdf_local = "" if args.use_cvmfs_lhapdf:
def simulation(args): gf.print_flush(args) print("Physical cores:", psutil.cpu_count(logical=False)) print("Total cores:", psutil.cpu_count(logical=True)) print("=" * 20, "Memory Information", "=" * 20) # get the memory details svmem = psutil.virtual_memory() print(f"Total: {get_size(svmem.total)}") print(f"Available: {get_size(svmem.available)}") print(f"Used: {get_size(svmem.used)}") print(f"Percentage: {svmem.percent}%") pid = os.getpid() py = psutil.Process(pid) memoryUse = py.memory_info()[0] # initialise world from file gf.print_flush("Initialising world...") world_file = "{}.hdf5".format(args.world) world = generate_world_from_hdf5(world_file, chunk_size=1_000_000) gf.print_flush("World loaded successfully...") geography = load_geography_from_hdf5(world_file) # leisure gf.print_flush("Initialising leisure...") world.pubs = Pubs.for_geography(geography) world.cinemas = Cinemas.for_geography(geography) world.groceries = Groceries.for_super_areas(geography.super_areas) # cemeteries gf.print_flush("Initialising cemeteries...") world.cemeteries = Cemeteries() # commute gf.print_flush("Initialising commute...") world.initialise_commuting() # infection selector gf.print_flush("Selecting infection...") selector = InfectionSelector.from_file() interaction = ContactAveraging.from_file(selector=selector) lhs_array = np.load("lhs_array.npy") parameters = generate_parameters_from_lhs(lhs_array, args.idx) interaction = set_interaction_parameters(parameters, interaction) gf.print_flush("Betas = {}, alpha = {}".format(interaction.beta, interaction.alpha_physical)) if not os.path.exists(SAVE_PATH): os.makedirs(SAVE_PATH) # save out parameters for later with open(SAVE_PATH + '/parameters.json', 'w') as f: json.dump(parameters, f) # seed infections seed = Seed.from_file( super_areas=world.super_areas, selector=selector, ) print(f"Memory used by JUNE's world: {get_size(memoryUse)}") simulator = Simulator.from_file(world, interaction, selector, seed=seed, config_filename=CONFIG_PATH, save_path=SAVE_PATH) simulator.run() # read = ReadLogger(SAVE_PATH) # world_df = read.world_summary() # ages_df = read.age_summary([0,10,20,30,40, # 50,60,70,80,90,100]) # loc_df = read.get_locations_infections() # r_df = read.get_r() # world_df.to_csv(SAVE_PATH + '/world_df.csv') # ages_df.to_csv(SAVE_PATH + '/ages_df.csv') # loc_df.to_csv(SAVE_PATH + '/loc_df.csv') # r_df.to_csv(SAVE_PATH + '/r_df.csv') gf.print_flush("Simulation finished!!!!") return None
def simulation(args): gf.print_flush(args) msoaslist = [ "E02005702", "E02005704", "E02005736", "E02005734", "E02001697", "E02001701", "E02001704", "E02001702", "E02001812", "E02001803", "E02001806", "E02001796", "E02001801", "E02001802", "E02001795", "E02001818", "E02001821", "E02001814", "E02001808", "E02001817", "E02001816", "E02001819", "E02001813", "E02001804", "E02001811", "E02001805", "E02001791", "E02001794", "E02001792", "E02004320", "E02004321", "E02004322", "E02004325", "E02004327", "E02004329", "E02004330", "E02004328", "E02001798", "E02001793", "E02005706", "E02002496", "E02002498", "E02002500", "E02002503", "E02002504", "E02002515", "E02002516", "E02006910", "E02002518", "E02002519", "E02002513", "E02002550", "E02002555", "E02002549", "E02002542", "E02002547", "E02002545", "E02002543", "E02002537", "E02002544", "E02002541", "E02002523", "E02002540", "E02002536", "E02002538", "E02002535", "E02006909", "E02002489", "E02002484", "E02002487", "E02002485", "E02002483", "E02002493", "E02002490", "E02002492", "E02002494", "E02002488", "E02002491", "E02004332", "E02002505", "E02002497", "E02002502", "E02006812", "E02002499", "E02002506", "E02006811", "E02002509", "E02002501", "E02002508", "E02002507", "E02002529", "E02002514", "E02002512" ] gf.print_flush("Generating world from msoalist...") geography = Geography.from_file({"msoa": msoaslist}) print('memory % used:', psutil.virtual_memory()[2]) geography.hospitals = Hospitals.for_geography(geography) geography.schools = Schools.for_geography(geography) geography.companies = Companies.for_geography(geography) geography.care_homes = CareHomes.for_geography(geography) demography = Demography.for_geography(geography) gf.print_flush("Geography and demography generated...") world = World(geography, demography, include_households=True, include_commute=False) gf.print_flush("World generated...") # leisure world.cinemas = Cinemas.for_geography(geography) world.pubs = Pubs.for_geography(geography) world.groceries = Groceries.for_super_areas(world.super_areas, venues_per_capita=1 / 500) gf.print_flush("Initialised leisure...") # commute world.initialise_commuting() gf.print_flush("Initialised commute...") # cemeteries world.cemeteries = Cemeteries() gf.print_flush("Initialised cemeteries...") # infection selector selector = InfectionSelector.from_file() interaction = ContactAveraging.from_file(selector=selector) gf.print_flush("Infection selected...") # define groups for betas groups = { "leisure": ['pub', 'grocery', 'cinema'], "commute": ['commute_unit', 'commute_city_unit', 'travel_unit'], "hospital": ['hospital'], "care_home": ['care_home'], "company": ['company'], "school": ['school'], "household": ['household'] } # define problem for latin hypercube sampling problem = { 'num_vars': len(groups), 'names': list(groups.keys()), 'bounds': [[1, 2] for _ in range(len(groups))] } lhs = latin.sample(problem, N=args.num_runs, seed=1)[args.idx] betas = {} for i, key in enumerate(groups): for sub_key in groups[key]: betas[sub_key] = lhs[i] # save out betas for later with open(SAVE_PATH + '/betas.json', 'w') as f: json.dump(betas, f) # set betas in simulation to sampled ones for key in betas: interaction.beta[key] = betas[key] gf.print_flush(interaction.beta) # seed infections seed = Seed( world.super_areas, selector, ) n_cases = int(len(world.people) / 10) seed.unleash_virus(n_cases) simulator = Simulator.from_file(world, interaction, selector, config_filename=CONFIG_PATH, save_path=SAVE_PATH) print('memory % used:', psutil.virtual_memory()[2]) simulator.run() gf.print_flush("Simulation finished!!!!") return None
if args.Sockets: nnlojet_command, socket_config = setup_sockets( args, nnlojet_command, bring_status) if args.Production: # Assume sockets does not work with production nnlojet_command += " -iseed {0}".format(args.seed) if gf.DEBUG_LEVEL > 1: gf.do_shell("ls") gf.do_shell("ldd -v {0}".format(args.executable)) # Run executable status_nnlojet = run_executable(nnlojet_command) # Store output status_copy, status_tar = store_output( args, socketed=args.Sockets, socket_config=socket_config) print_copy_status(args, status_copy) if args.Sockets: try: # only the first one arriving will go through! gf.print_flush("Close Socket connection") _ = socket_sync_str(args.Host, args.port, "bye!") # Be polite except socket.error: pass statuses = [status_nnlojet, status_copy, status_tar, bring_status] status_code = int(any(status != 0 for status in statuses)) gf.end_program(status_code)
def download_world(input_folder, world): world_file = world + ".hdf5" gf.print_flush("downloading " + input_folder + "/worlds/" + world_file) stat = gf.copy_from_grid(input_folder + "/worlds/" + world_file, world_file, args) return stat
status += gf.run_command( "source JUNE_env/bin/activate && export PYTHONPATH=$PYTHONPATH:`pwd`/JUNE/ && ./{executable} --idx={idx} --world={world} --latin_hypercube={latin_hypercube} {runcard} {outfile}" .format(executable=args.executable, idx=args.seed, world=args.world, latin_hypercube=args.latin_hypercube, runcard=args.runname, outfile="{0}.out".format(args.seed))) return status # ------------------------- MAIN ------------------------- if __name__ == "__main__": # Generic startup: start_time = datetime.datetime.now() gf.print_flush("Start time: {0}".format( start_time.strftime("%d-%m-%Y %H:%M:%S"))) args = parse_arguments() # lhapdf_local = "" # if args.use_cvmfs_lhapdf: # lhapdf_local = args.cvmfs_lhapdf_location # set_environment(lhapdf_local) if gf.DEBUG_LEVEL > -1: # Architecture info gf.print_flush("Python version: {0}".format(sys.version)) gf.print_node_info("node_info.log") if args.copy_log: # initialise with node name