def create_base(): """ Set up for 19 events, 5 gmpe_max, 7 periods, 11 spawning, 3 rec mod. 2 Surfaces. 17277 sites """ sdp = ParameterData() # Operation Mode sdp.run_type = "hazard" sdp.is_scenario = False sdp.site_tag = "bench" sdp.return_periods = [ 100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900 ] sdp.input_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_input') sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', 'trial_matrix') sdp.use_site_indexes = True #False sdp.site_indexes = [1] sdp.event_control_tag = "use" sdp.zone_source_tag = "" # Probabilistic input sdp.prob_number_of_events_in_zones = [5, 5, 5, 4] # Attenuation sdp.atten_collapse_Sa_of_atten_models = False #True sdp.atten_variability_method = 1 sdp.atten_spawn_bins = 11 sdp.atten_periods = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9, 1.0] sdp.atten_threshold_distance = 400 sdp.atten_override_RSA_shape = None sdp.atten_cutoff_max_spectral_displacement = False sdp.atten_pga_scaling_cutoff = 2 sdp.atten_smooth_spectral_acceleration = None # Amplification sdp.use_amplification = True sdp.amp_variability_method = None sdp.amp_min_factor = 0.6 sdp.amp_max_factor = 10000 # Save sdp.save_hazard_map = True sdp.save_motion = False return sdp
def create_base(): """ Set up for 19 events, 5 gmpe_max, 7 periods, 11 spawning, 3 rec mod. 2 Surfaces. 17277 sites """ sdp = ParameterData() # Operation Mode sdp.run_type = "hazard" sdp.is_scenario = False sdp.site_tag = "bench" sdp.return_periods = [100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900] sdp.input_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_input') sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','trial_matrix') sdp.use_site_indexes = True #False sdp.site_indexes = [1] sdp.event_control_tag = "use" sdp.zone_source_tag = "" # Probabilistic input sdp.prob_number_of_events_in_zones = [5,5,5,4] # Attenuation sdp.atten_collapse_Sa_of_atten_models = False #True sdp.atten_variability_method = 1 sdp.atten_spawn_bins = 11 sdp.atten_periods = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9, 1.0] sdp.atten_threshold_distance = 400 sdp.atten_override_RSA_shape = None sdp.atten_cutoff_max_spectral_displacement = False sdp.atten_pga_scaling_cutoff = 2 sdp.atten_smooth_spectral_acceleration = None # Amplification sdp.use_amplification = False #True sdp.amp_variability_method = None sdp.amp_min_factor = 0.6 sdp.amp_max_factor = 10000 # Save sdp.save_hazard_map = True sdp.save_motion = False return sdp
def old_max_simulation(): """ attribute # JSON attribute events 8,000,000* len_events Sites 3,000,000 len_block_sites periods 10 len_atten_periods return periods 10 len_return_periods source zones 150 len_source_zones SA surfaces (Sa soil and Sa bed rock) 2 len_SA_surfaces # of GM models 5 len_max_GMPEs Note, i'm not doing 150 source zones. The mem estimate's have been good, without taking it into account. I ran this on rhe-compute, but since there are so many sites it would have taken an age to finish, so I stopped it before completion. It had started cycling over the sites without a memory error """ runs = [] num_sources = 4 sdp = create_base() ### sdp.site_indexes = range(1, 400) events = 800 events = 8000000 events = 80000 #events = 80 sdp.return_periods = [100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900, 1000.] sdp.atten_periods = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] sdp.use_amplification = True sdp.atten_spawn_bins = 1 sites = 160 sites = 1600 sdp.file_parallel_log_level = 'debug' ### events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.use_site_indexes = True setof15k = sites/15000 if setof15k > 0: sdp.site_indexes = range(1, 15001) * setof15k else: sdp.site_indexes = range(1, sites) print "len(sdp.site_indexes)", len(sdp.site_indexes) sdp.event_control_tag = "4GMPE" dir_last = 'old_max_sim_test_' + str(sum( sdp.prob_number_of_events_in_zones)) + \ "_sites" + str(len(sdp.site_indexes)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_reduce_mem', dir_last) runs.append({"processes":40, "sdp":sdp}) return runs
def old_max_simulation(): """ attribute # JSON attribute events 8,000,000* len_events Sites 3,000,000 len_block_sites periods 10 len_atten_periods return periods 10 len_return_periods source zones 150 len_source_zones SA surfaces (Sa soil and Sa bed rock) 2 len_SA_surfaces # of GM models 5 len_max_GMPEs Note, i'm not doing 150 source zones. The mem estimate's have been good, without taking it into account. """ runs = [] num_sources = 4 sdp = create_base() ### sdp.site_indexes = range(1, 400) events = 80000 events = 800 events = 8000000 events = 80 sdp.return_periods = [100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900, 1000.] sdp.atten_periods = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] sdp.use_amplification = True sdp.atten_spawn_bins = 1 sites_needed = 17278 sdp.file_parallel_log_level = 'debug' sdp.use_site_indexes = False ### num_sites = 17277 events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.use_site_indexes = True sdp.site_indexes = create_sites_list(num_sites, sites_needed) assert sum(sdp.prob_number_of_events_in_zones) == events assert len(sdp.site_indexes) == sites_needed sdp.event_control_tag = "4GMPE" dir_last = 'old_max_sim_again_' + str(sum( sdp.prob_number_of_events_in_zones)) + \ "_sites" + str(len(sdp.site_indexes)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) runs.append({"processes":1, "sdp":sdp}) return runs
def create_meta_log(): path = os.path.join(eqrm_data_home(), 'test_national', 'memory_output') # 'memory_output_riskB') log_pairs = build_log_info(path) #print "len(log_pairs)", len(log_pairs) log_pairs_estimate_mem(log_pairs) mem_pairs = [] for alog in log_pairs: just_mem = {} for key, value in alog.iteritems(): if "mem" in key: just_mem[key] = value mem_pairs.append(just_mem) write_meta_log(mem_pairs, os.path.join(path, 'meta_log.csv'))
def build_runs_list_large_standard(): runs = [] num_sources = 4 sdp = create_base() sdp.site_indexes = range(1, 400) sdp.prob_number_of_events_in_zones = [20000] * 4 sdp.atten_periods = [0.0, 0.3, 1.0] sdp.atten_spawn_bins = 5 sdp.event_control_tag = "4GMPE" dir_last = 'big_standard_' + str(sum( sdp.prob_number_of_events_in_zones)) + \ "_sites" + str(len(sdp.site_indexes)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) runs.append({"processes":1, "sdp":sdp}) return runs
def risk_simulation(): """ """ runs = [] num_sources = 6 sdp = create_base() ### sdp.run_type = "risk" sdp.atten_collapse_Sa_of_atten_models = True sdp.save_total_financial_loss = True sdp.save_hazard_map = True sdp.site_indexes = range(1, 400) sdp.site_indexes = range(1, 5) events = 80000 events = 80 sdp.return_periods = [100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900, 1000.] sdp.atten_periods = [0.1, 0.2, 1.0] sdp.atten_variability_method = 2 sdp.site_tag = "newc" #sdp.file_parallel_log_level = 'debug' ### events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.use_site_indexes = True #sdp.event_control_tag = "4GMPE" dir_last = 'initial_risk_' + str(sum( sdp.prob_number_of_events_in_zones)) + \ "_sites" + str(len(sdp.site_indexes)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) runs.append({"processes":1, "sdp":sdp}) return runs
def build_runs_list(): runs = [] num_sources = 4 # The base run sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','trial_matrix') runs.append({"processes":1, "sdp":sdp}) # Testing parallel sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','trial_parallel') sdp.site_indexes = [1, 2] runs.append({"processes":2, "sdp":sdp}) # Test more events sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','more_events') sdp.site_indexes = [1, 2,3,4] sdp.prob_number_of_events_in_zones = \ [x*4 for x in sdp.prob_number_of_events_in_zones] runs.append({"processes":2, "sdp":sdp}) # Test small and larger event set base = [5,5,5,4] event_lists = [base, [x*200 for x in base]] for event_list in event_lists: # Test more rec mod sdp = create_base() dir_last = 'more_rec_mod_events' + str(sum(event_list)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) sdp.site_indexes = [1] sdp.prob_number_of_events_in_zones = event_list sdp.zone_source_tag = "rec_mod" runs.append({"processes":1, "sdp":sdp}) # Testing more GMPE's sdp = create_base() dir_last = 'more_gmpe_events' + str(sum(event_list)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) sdp.site_indexes = [1] sdp.prob_number_of_events_in_zones = event_list sdp.event_control_tag = "GMPE" runs.append({"processes":1, "sdp":sdp}) # Testing more periods's sdp = create_base() dir_last = 'more_periods_events' + str(sum(event_list)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) sdp.site_indexes = [1] sdp.prob_number_of_events_in_zones = event_list sdp.atten_periods = arange(0,2.1, 0.1) runs.append({"processes":1, "sdp":sdp}) # Test more spawning sdp = create_base() dir_last = 'more_spawning_events' + str(sum(event_list)) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_last) sdp.site_indexes = [1] sdp.prob_number_of_events_in_zones = event_list sdp.atten_variability_method = 1 sdp.atten_spawn_bins = 44 runs.append({"processes":1, "sdp":sdp}) # spawning all sites sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','bigger_spawning') events = 50 events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.atten_variability_method = 1 sdp.atten_spawn_bins = 100 sdp.use_site_indexes = False runs.append({"processes":1, "sdp":sdp}) # increasing events, 2 sites for events in [1000, 10000, 100000]: sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','events_' + str(events)) events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.site_indexes = [1, 2] runs.append({"processes":1, "sdp":sdp}) # Testing all sites sdp = create_base() sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','trial_all_sites') sdp.use_site_indexes = False runs.append({"processes":2, "sdp":sdp}) # increasing events, all sites for events in [1000, 5000]: #, 100000]: for proc in [1]: sdp = create_base() sdp.site_indexes = range(1, 500) dir_name = 'sites_' + str(len(sdp.site_indexes)) + '_events_' + \ str(events) + \ '_' + 'proc_'+ \ str(proc) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output', dir_name) events_source = events/num_sources sdp.prob_number_of_events_in_zones = [events_source]*num_sources runs.append({"processes":proc, "sdp":sdp}) return runs
def risk_simulation(): """ """ runs = [] num_sources = 6 ### sdp = create_base_risk() sdp.atten_collapse_Sa_of_atten_models = True sdp.atten_variability_method = 2 ### sites_needed = 1 events = 80 ### events_source = events/num_sources num_sites = 17278 sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.site_indexes = create_sites_list(num_sites, sites_needed) dir_last = 'initial_risk_' + str(events) + \ "_sites" + str(sites_needed) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output_riskB', dir_last) runs.append({"processes":1, "sdp":sdp}) sdp = create_base_risk() sdp.atten_collapse_Sa_of_atten_models = True sdp.atten_variability_method = 2 ### sites_needed = 2 events = 80 ### events_source = events/num_sources num_sites = 17278 sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.site_indexes = create_sites_list(num_sites, sites_needed) dir_last = 'initial_risk_' + str(events) + \ "_sites" + str(sites_needed) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output_riskB', dir_last) runs.append({"processes":1, "sdp":sdp}) sdp = create_base_risk() sdp.atten_collapse_Sa_of_atten_models = True sdp.atten_variability_method = 2 ### sites_needed = 1 events = 800 ### events_source = events/num_sources num_sites = 17278 sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.site_indexes = create_sites_list(num_sites, sites_needed) dir_last = 'initial_risk_' + str(events) + \ "_sites" + str(sites_needed) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output_risk', dir_last) #runs.append({"processes":1, "sdp":sdp}) sdp = create_base_risk() sdp.atten_collapse_Sa_of_atten_models = True sdp.atten_variability_method = 2 ### sites_needed = 10 events = 800 ### events_source = events/num_sources num_sites = 17278 sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.site_indexes = create_sites_list(num_sites, sites_needed) dir_last = 'initial_risk_' + str(events) + \ "_sites" + str(sites_needed) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output_risk', dir_last) #runs.append({"processes":1, "sdp":sdp}) sdp = create_base_risk() sdp.atten_collapse_Sa_of_atten_models = True sdp.atten_variability_method = 2 ### sites_needed = 1 events = 80000 ### events_source = events/num_sources num_sites = 17278 sdp.prob_number_of_events_in_zones = [events_source]*num_sources sdp.prob_number_of_events_in_zones[0] += events%num_sources sdp.site_indexes = create_sites_list(num_sites, sites_needed) dir_last = 'initial_risk_' + str(events) + \ "_sites" + str(sites_needed) sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output_risk', dir_last) #runs.append({"processes":1, "sdp":sdp}) return runs
Run multipe EQRM simulations, to get time and memory metrics. """ import subprocess import os from scipy import arange import eqrm_code.util from eqrm_code import util from eqrm_code.parse_in_parameters import eqrm_data_home, get_time_user, \ ParameterData, create_parameter_data, eqrm_flags_to_control_file from eqrm_code.analysis import main from eqrm_code.estimator import log_estimate_memory_use ROOTOUTPUTDIR = os.path.join(eqrm_data_home(), 'test_national', 'reduce_mem') class ListLengthError(Exception): pass def create_base(): """ Set up for 19 events, 5 gmpe_max, 7 periods, 11 spawning, 3 rec mod. 2 Surfaces. 17277 sites """ sdp = ParameterData()
from os.path import join # Operation Mode run_type = "hazard" is_scenario = False site_tag = "bench" zone_source_tag = "" # General return_periods = [100, 200, 300, 400, 500, 600, 700, 800, 900] input_dir = None output_dir = None use_site_indexes = True # False site_indexes = [1] event_control_tag = "use" input_dir = join(eqrm_data_home(), "test_national", "memory_input") output_dir = join(eqrm_data_home(), "test_national", "memory_output", get_time_user()) prob_number_of_events_in_zones = [1e8, 1e8, 1e8, 1e8] prob_number_of_events_in_zones = [1, 1, 1, 1] # prob_number_of_events_in_zones = [100000,100000,100000,100000] # Attenuation atten_collapse_Sa_of_atten_models = 1 atten_variability_method = 2 atten_periods = [0.0, 0.2, 1.0] atten_threshold_distance = 400 atten_override_RSA_shape = None atten_cutoff_max_spectral_displacement = False atten_pga_scaling_cutoff = 2
def create_meta_log(path=None): if path == None: path = os.path.join(eqrm_data_home(), 'test_national', 'reduce_mem') log_pairs = build_log_info(path) log_pairs_estimate_mem(log_pairs)
def create_base_risk(): """ Using newcastle data. """ sdp = ParameterData() # Operation Mode sdp.run_type = "risk_csm" sdp.is_scenario = False sdp.site_tag = "newc" sdp.atten_periods = [0.1, 0.2, 1.0] sdp.return_periods = [100.75, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900] sdp.input_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_input') sdp.output_dir = os.path.join(eqrm_data_home(), 'test_national', 'memory_output','trial_matrix') sdp.use_site_indexes = True #False sdp.site_indexes = [1] sdp.event_control_tag = "use" sdp.zone_source_tag = "" # Probabilistic input sdp.prob_number_of_events_in_zones = [5,5,5,4] # Attenuation sdp.atten_collapse_Sa_of_atten_models = False #True sdp.atten_variability_method = 1 sdp.atten_spawn_bins = 11 sdp.atten_periods = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9, 1.0] sdp.atten_threshold_distance = 400 sdp.atten_override_RSA_shape = None sdp.atten_cutoff_max_spectral_displacement = False sdp.atten_pga_scaling_cutoff = 2 sdp.atten_smooth_spectral_acceleration = None # Amplification sdp.use_amplification = False #True sdp.amp_variability_method = None sdp.amp_min_factor = 0.6 sdp.amp_max_factor = 10000 # Buildings sdp.buildings_usage_classification = "FCB" sdp.buildings_set_damping_Be_to_5_percent = False # Capacity Spectrum Method sdp.csm_use_variability = False sdp.csm_variability_method = None sdp.csm_standard_deviation = 0.3 sdp.csm_damping_regimes = 0 sdp.csm_damping_modify_Tav = True sdp.csm_damping_use_smoothing = True sdp.csm_hysteretic_damping = "curve" sdp.csm_SDcr_tolerance_percentage = 1 sdp.csm_damping_max_iterations = 7 # Loss sdp.loss_min_pga = 0.05 sdp.loss_regional_cost_index_multiplier = 1.4516 sdp.loss_aus_contents = 0 # Save sdp.save_hazard_map = True sdp.save_total_financial_loss = True sdp.save_hazard_map = True return sdp
from os.path import join # Operation Mode run_type = "hazard" is_scenario = False site_tag = "bench" zone_source_tag = "" # General return_periods = [100, 200, 300, 400, 500, 600, 700, 800, 900] input_dir = None output_dir = None use_site_indexes = True #False site_indexes = [1] event_control_tag = "use" input_dir = join(eqrm_data_home(), 'test_national', 'memory_input') output_dir = join(eqrm_data_home(), 'test_national', 'memory_output', get_time_user()) prob_number_of_events_in_zones = [1e8, 1e8, 1e8, 1e8] prob_number_of_events_in_zones = [1, 1, 1, 1] #prob_number_of_events_in_zones = [100000,100000,100000,100000] # Attenuation atten_collapse_Sa_of_atten_models = 1 atten_variability_method = 2 atten_periods = [0.0, 0.2, 1.0] atten_threshold_distance = 400 atten_override_RSA_shape = None atten_cutoff_max_spectral_displacement = False
from eqrm_code.parse_in_parameters import eqrm_data_home, get_time_user from os.path import join # Operation Mode run_type = "hazard" is_scenario = False site_tag = "bench" # General return_periods = [100, 200, 300, 400, 500, 600, 700, 800, 900] input_dir = None output_dir = None use_site_indexes = False site_indexes = [1, 2, 3, 4, 5] event_control_tag = "use" input_dir = join(eqrm_data_home(), 'test_national', 'memory_input') output_dir = join(eqrm_data_home(), 'test_national', 'memory_output', get_time_user()) prob_number_of_events_in_zones = [10000, 10000, 10000, 10000] #prob_number_of_events_in_zones = [100, 100, 100, 100] # Attenuation atten_collapse_Sa_of_atten_models = 1 atten_variability_method = 2 atten_periods = [0.0, 0.2, 1.0] atten_threshold_distance = 400 atten_override_RSA_shape = None atten_cutoff_max_spectral_displacement = False