def build_joint_pafs_single_location(drmaa, queue: str, jobs: Dict, location: str, draws, output_dir: Path, session): sanitized_location = sanitize_location(location) path = output_dir / sanitized_location if path.exists() and len(draws) == 1000: shutil.rmtree(path) path.mkdir(exist_ok=True, mode=0o775) for draw in draws: job_template = session.createJobTemplate() job_template.remoteCommand = shutil.which("python") job_template.outputPath = f":{output_dir}/output_logs" job_template.errorPath = f":{output_dir}/error_logs" job_template.args = [__file__, str(path), f'"{location}"', draw] job_template.nativeSpecification = ( f'-V ' # Export all environment variables f'-b y ' # Command is a binary (python) f'-P {project_globals.CLUSTER_PROJECT} ' f'-q {queue} ' f'-l fmem={project_globals.MAKE_ARTIFACT_MEM} ' f'-l fthread={project_globals.MAKE_ARTIFACT_CPU} ' f'-l h_rt={project_globals.MAKE_ARTIFACT_RUNTIME} ' f'-l archive=TRUE ' # Need J-drive access for data f'-N {sanitize_location(location)}_joint_pafs') # Name of the job jobs[location] = (session.runJob(job_template), drmaa.JobState.UNDETERMINED) logger.info( f'Submitted job {jobs[location][0]} to build joint pafs for {location} and draw {draw}.' ) session.deleteJobTemplate(job_template)
def make_sample_history_single_location(drmaa, queue: str, jobs: Dict, location: str, scenarios: List[str], output_dir: Path, session): sanitized_location = sanitize_location(location) path = output_dir / sanitized_location if not path.exists(): path.mkdir(exist_ok=True, mode=0o775) (path / 'error_logs').mkdir(exist_ok=True, mode=0o775) (path / 'output_logs').mkdir(exist_ok=True, mode=0o775) for scenario in scenarios: job_template = session.createJobTemplate() job_template.remoteCommand = shutil.which("python") job_template.outputPath = f":{path}/output_logs" job_template.errorPath = f":{path}/error_logs" job_template.args = [__file__, str(path), f'"{sanitized_location}"', scenario] job_template.nativeSpecification = (f'-V ' # Export all environment variables f'-b y ' # Command is a binary (python) f'-P {project_globals.CLUSTER_PROJECT} ' f'-q {queue} ' f'-l fmem={project_globals.MAKE_ARTIFACT_MEM} ' f'-l fthread={project_globals.MAKE_ARTIFACT_CPU} ' f'-l h_rt={project_globals.MAKE_ARTIFACT_RUNTIME} ' f'-l archive=TRUE ' # Need J-drive access for data f'-N {sanitize_location(location)}_{scenario}_sample_history') # Job name jobs[location] = (session.runJob(job_template), drmaa.JobState.UNDETERMINED) logger.info(f'Submitted job {jobs[location][0]} to generate sample history ' f'for {location} and scenario {scenario}.') session.deleteJobTemplate(job_template)
def sample_raw_rx_change(location: str, draw: int, rx_change: str) -> float: location = sanitize_location(location) """Raw result: needs to be adjusted""" seed = get_hash(f'{rx_change}_probability_draw_{draw}_location_{location}') data = pd.read_csv(paths.PROB_ADDING_DRUGS).set_index('probability_type') params = data.loc[rx_change, :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def sample_probability_target_given_rx(location: str, draw: int) -> float: location = sanitize_location(location) seed = get_hash( f'target_given_rx_probability_draw_{draw}_location_{location}') data = pd.read_csv(paths.PROB_TARGET_GIVEN_RX).set_index(LOCATION_COLUMN) params = data.loc[location, :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def sample_adherence(location: str, draw: int, multi_pill: bool, previous_cve: bool) -> float: location = sanitize_location(location) seed = get_hash(f'adherence_probability_draw_{draw}_location_{location}') data = pd.read_csv(paths.ADHERENCE_PARAMETERS).set_index( [LOCATION_COLUMN, 'multi_pill', 'previous_cve']) params = data.loc[(location, int(multi_pill), int(previous_cve)), :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def sample_probability_testing_ldl_c(location: str, draw: int) -> float: location = sanitize_location(location) seed = get_hash( f'testing_ldl_c_probability_draw_{draw}_location_{location}') data = pd.read_csv( paths.PROB_TESTING_LDL_C_PATH).set_index(LOCATION_COLUMN) params = data.loc[location, :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def build_artifacts(location: str, output_dir: str, append: bool, verbose: int): """Main application function for building artifacts. Parameters ---------- location The location to build the artifact for. Must be one of the locations specified in the project globals or the string 'all'. If the latter, this application will build all artifacts in parallel. output_dir The path where the artifact files will be built. append Whether we should append to existing artifacts at the given output directory. Has no effect if artifacts are not found. verbose How noisy the logger should be. """ output_dir = Path(output_dir) if location in project_globals.LOCATIONS: path = Path(output_dir) / f'{sanitize_location(location)}.hdf' if path.exists() and not append: click.confirm( f"Existing artifact found for {location}. Do you want to delete and rebuild?", abort=True) logger.info(f'Deleting artifact at {str(path)}.') path.unlink() build_single_location_artifact(path, location) elif location == 'all': # FIXME: could be more careful existing_artifacts = set([ item.stem for item in output_dir.iterdir() if item.is_file() and item.suffix == '.hdf' ]) locations = set( [sanitize_location(loc) for loc in project_globals.LOCATIONS]) existing = locations.intersection(existing_artifacts) if existing and not append: click.confirm( f'Existing artifacts found for {existing}. Do you want to delete and rebuild?', abort=True) for loc in existing: path = output_dir / f'{loc}.hdf' logger.info(f'Deleting artifact at {str(path)}.') path.unlink() build_all_artifacts(output_dir, verbose) else: raise ValueError( f'Location must be one of {project_globals.LOCATIONS} or the string "all". ' f'You specified {location}.')
def sample_raw_drug_prescription(location: str, draw: int, drug: str) -> float: location = sanitize_location(location) """Raw result: needs to be adjusted""" seed = get_hash( f'{drug}_prescription_probability_draw_{draw}_location_{location}') data = pd.read_csv(paths.CURRENT_RX_DATA_PATH).set_index( [LOCATION_COLUMN, 'current_prescription']) params = data.loc[(location, drug.replace('_', ' ')), :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def sample_therapy_type(location: str, draw: int, therapy_type: str) -> float: location = sanitize_location(location) therapy_type = therapy_type.upper( ) if therapy_type is FDC else therapy_type seed = get_hash( f'{therapy_type}_probability_draw_{draw}_location_{location}') data = pd.read_csv(paths.PROB_THERAPY_TYPE).set_index( [LOCATION_COLUMN, 'therapy_type']) params = data.loc[(location, therapy_type), :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def sample_probability_increasing_dose(scenario: str, location: str, draw: int) -> float: location = sanitize_location(location) scenario = scenario if scenario == 'baseline' else 'intervention' seed = get_hash( f'target_given_rx_probability_scenario_{scenario}_draw_{draw}_location_{location}' ) data = pd.read_csv(paths.PROB_ADDING_DRUGS).set_index( [LOCATION_COLUMN, 'scenario']) params = data.loc[(location, scenario), :] return sample_truncnorm_distribution(seed, params[MEAN_COLUMN], params[SD_COLUMN])
def build_joint_pafs(location: str, draws: str, verbose: int, queue: str): # Local import to avoid data dependencies from vivarium_inputs import globals as vi_globals, utilities output_dir = paths.JOINT_PAF_DIR locations = project_globals.LOCATIONS if location == 'all' else [location] from vivarium_cluster_tools.psimulate.utilities import get_drmaa drmaa = get_drmaa() jobs = {} draw_list = {'all': range(1000), 'none': []}.get(draws, draws.split(',')) with drmaa.Session() as session: for location in locations: build_joint_pafs_single_location(drmaa, queue, jobs, location, draw_list, output_dir, session) if verbose: logger.info('Entering monitoring loop.') logger.info('-------------------------') logger.info('') while any([ job[1] not in [drmaa.JobState.DONE, drmaa.JobState.FAILED] for job in jobs.values() ]): for location, (job_id, status) in jobs.items(): jobs[location] = (job_id, session.jobStatus(job_id)) logger.info( f'{location:<35}: {decode_status(drmaa, jobs[location][1]):>15}' ) logger.info('') time.sleep(project_globals.MAKE_ARTIFACT_SLEEP) logger.info('Checking status again') logger.info('---------------------') logger.info('') for location in locations: logger.info(f'Merging data for location - {location}') sanitized_location = sanitize_location(location) location_dir = paths.JOINT_PAF_DIR / sanitized_location existing_data_path = output_dir / f'{sanitized_location}.hdf' joint_pafs = [] if existing_data_path.exists(): joint_pafs.append( pd.read_hdf(output_dir / f'{sanitized_location}.hdf')) joint_pafs[0].to_hdf(output_dir / f'{sanitized_location}-old.hdf', 'data') for file_path in location_dir.iterdir(): draw = file_path.parts[-1].split('.')[0] draw_joint_paf = pd.read_hdf(file_path).rename(columns={0: draw}) draw_joint_paf['affected_measure'] = 'incidence_rate' draw_joint_paf = draw_joint_paf.set_index( list(draw_joint_paf.columns.drop(draw))) joint_pafs.append(draw_joint_paf) joint_paf_data = pd.concat(joint_pafs, axis=1) joint_paf_data = joint_paf_data[ vi_globals.DRAW_COLUMNS] # sort the columns joint_paf_data = utilities.sort_hierarchical_data( joint_paf_data).convert_objects() joint_paf_data.to_hdf(output_dir / f'{sanitized_location}.hdf', 'data') shutil.rmtree(location_dir) logger.info('**Done**')