Ejemplo n.º 1
0
        for sim_id, sim in calib_data.iteritems():
        
            sim_output = sim['output']
            
            x_temp_h = float(sim['meta']['x_Temporary_Larval_Habitat'])
            
            const_h_struct = ast.literal_eval(sim['meta']['scale_larval_habitats_single'])
            const_h = const_h_struct[0][1][1]
            
            itn_level_struct = ast.literal_eval(sim['meta']['add_ITN_mult'])
            itn_level = itn_level_struct[0][1][0][0][1]
                   
            drug_coverage_level_struct = ast.literal_eval(sim['meta']['add_drug_multi_campaigns'])
            drug_coverage_level = drug_coverage_level_struct[0][1][0]['coverage']
            
            sim_key = get_sim_key(x_temp_h, const_h, itn_level, drug_coverage_level)
            sim_group_key =  get_sim_group_key(itn_level, drug_coverage_level)
         
        
            sim_data = sim_output['Channels']['New Clinical Cases']['Data']
            
            
            if not sim_group_key in cc:
                cc[sim_group_key] = {}

            cc[sim_group_key][sim_key] = sim_data
            
        
        del calib_data
        gc.collect()
def get_sweep_results(sim_meta_config_files, calib_file_path, tags_data_file_path):
    
    # Login to COMPs
    comps_login()
    
    
    # find total number of simulations across given experiment files
    num_sims = 0
    for sim_meta_config_file in sim_meta_config_files:
        with open(sim_meta_config_file) as metadata_file:
             metadata = json.loads(metadata_file.read())

        num_sims = num_sims +  len(metadata['sims'])
    
    

    # Download simulations locally
    # sample sim meta config file (like "C:\\Users\\Mnikolov\\Zambia-raw\\dtk-scripts\\1node\\simulations\\Sinamalina_Sinazongwe_Calibration_e9979059-33f8-e411-93f9-f0921c16b9e7.json")
    #print 'Downloading simulations from experiment ' + str(sim_meta_config_files) + '...'

    
    # simulations tag data structure: accumulates sims meta information from sims tags
    tag_data = {
                        'ITN trajectory': [],\
                        'Drug coverage per round': [],\
                        'Temporary habitat scale': [],\
                        'Constant habitat scale': []\
                }
    
    
    # iterate through experiments
    calib_output = {}    
    
    # count processed sims to updated progress
    count = 0
    for sim_meta_config_file in sim_meta_config_files:
        
        
        # construct experiment directory structure
        with open(sim_meta_config_file) as metadata_file:
             metadata = json.loads(metadata_file.read())
    
        output_path = metadata['sim_root']
        exp_id = metadata['exp_id']
        exp_name = metadata['exp_name']
        
        sim_dir_map  = CompsDTKOutputParser.createSimDirectoryMap(exp_id)
        
        # get all successfully completed sims in experiment
        for sim_id, sim in metadata['sims'].items():
            
            # get path to the sim timeseries channels data
            timeseries_path = os.path.join(sim_dir_map[sim_id],'output', 'InsetChart.json')

            
            #get sim timeseries channels data; json2dict returns None if timeseries_path points to non-existing file, which is the case if the sim has not successfully finished 
            sim_output = json2dict(timeseries_path)

            # only download successfully completed  simulations
            if sim_output == None:
                continue

            
            # delete all but the specified channels
            for channel in sim_output['Channels'].keys():
                if not channel in channels:
                   del(sim_output['Channels'][channel])
                          
            # process specified reports
            report_channels_data = {}
            if not reports_channels == None:
               report_channels_data = process_reports(reports_channels, sim_dir_map, sim_id)
            
            # record sim meta information including sim tags
            tags_path = os.path.join(sim_dir_map[sim_id], 'tags.json')
            f = open(tags_path, 'r')
            tags = f.read()
            sim_meta =  ast.literal_eval(tags)
            append_tag_data(sim_meta, tag_data)
        
            
            # construct sim group key and sim key
            x_temp_h = sim_meta_2_temp_h(sim_meta)
            const_h = sim_meta_2_const_h(sim_meta)
            itn_level = sim_meta_2_itn_level(sim_meta)
            drug_coverage_level = sim_meta_2_drug_cov(sim_meta)
            
            sim_key = get_sim_key(x_temp_h, const_h, itn_level, drug_coverage_level)
            sim_group_key =  get_sim_group_key(itn_level, drug_coverage_level)
        

            # store sim channels data  
            if sim_group_key not in calib_output:
                calib_output[sim_group_key] = {}

            calib_output[sim_group_key][sim_key] = {
                                                    # can add/remove data entries depending on needs
                                                    'prevalence': sim_output['Channels']['New Diagnostic Prevalence']['Data'],
                                                    'reinfections': report_channels_data['reinfections'],
                                                    'meta':sim_meta,
                                                    'sim_id':sim_id
                                                    }
    '''    
    count = count + 1
    percent_complete = 100*count/(num_sims+0.0)
    sys.stdout.write('\r')
    sys.stdout.write('%2f %%' % percent_complete)
    #sys.stdout.write('%d' % count)
    sys.stdout.flush()
    '''    
        
    print ""
    print "Writing files..."
    
    with open(calib_file_path, 'w') as calib_f:
            json.dump(calib_output, calib_f)
            print str(len(calib_output)) + ' simulation results saved to ' + calib_file_path
            
    with open(tags_data_file_path, 'w') as tags_f:
            json.dump(tag_data, tags_f)
            print 'Meta data tags saved to ' + tags_data_file_path
    
    print ""
    
    return calib_f