def setUpTestData(self): ''' Set up directory to test data and run the pipeline. ''' base_path = 'epoch-add-parallel-deruiter-forced' self.base_run = os.path.join( s.PIPELINE_WORKING_DIR, base_path ) # add image run make_testdir(self.base_run) gen_config.gen_config( base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02'] ) call_command('runpipeline', self.base_run) gen_config.gen_config( base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x', '12'] ) call_command('runpipeline', self.base_run) self.forced = {} for f in os.listdir(self.base_run): if f[:6] == 'forced' and f[-3:] != 'bak': self.forced[f] = pd.read_parquet( os.path.join(self.base_run, f) ) # remove directory shutil.rmtree(self.base_run)
def setUpTestData(self): ''' Set up directories to test data, run the pipeline, and read files. ''' base_path = 'epoch-add-parallel-deruiter' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) # run with add image make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02']) call_command('runpipeline', self.base_run) self.ass_backup = pd.read_parquet( os.path.join(self.base_run, 'associations.parquet')) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.base_run) self.sources = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations = pd.read_parquet( os.path.join(self.base_run, 'relations.parquet')) self.ass = pd.read_parquet( os.path.join(self.base_run, 'associations.parquet')) # remove test directory shutil.rmtree(self.base_run)
def setUpTestData(self): ''' Set up directories to test data, run the pipeline, and read files. ''' base_path = 'normal-advanced' compare_path = 'parallel-advanced' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path) # run with normal make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.base_run) self.sources_base = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations_base = pd.read_parquet( os.path.join(self.base_run, 'relations.parquet')) # run with parallel make_testdir(self.compare_run) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.compare_run) self.sources_compare = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations_compare = pd.read_parquet( os.path.join(self.compare_run, 'relations.parquet')) # remove directories shutil.rmtree(self.base_run) shutil.rmtree(self.compare_run)
def setUpTestData(self): ''' Set up directory to test data and run the pipeline. ''' base_path = 'epoch-advanced' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) # setup test directory make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.base_run) # read output self.sources = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations = pd.read_parquet( os.path.join(self.base_run, 'relations.parquet')) # remove test directory shutil.rmtree(self.base_run)
def setUpTestData(self): ''' Set up directories to test data, run the pipeline, and read the files. ''' base_path = 'normal-basic' compare_path = 'add-add-parallel-basic' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path) # run with all images make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x', '12']) call_command('runpipeline', self.base_run) self.sources_base = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations_base = pd.read_parquet( os.path.join(self.base_run, 'relations.parquet')) # run with add image make_testdir(self.compare_run) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02']) call_command('runpipeline', self.compare_run) self.ass_backup = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) # run with epoch 05 06 gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.compare_run) self.ass_backup_mid = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) # run with epoch 12 gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x', '12']) call_command('runpipeline', self.compare_run) self.ass_compare = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) self.sources_compare = pd.read_parquet( os.path.join(self.compare_run, 'sources.parquet')) self.relations_compare = pd.read_parquet( os.path.join(self.compare_run, 'relations.parquet')) # remove test directories shutil.rmtree(self.base_run) shutil.rmtree(self.compare_run)
def setUpTestData(self): ''' Set up directories to test data, run the pipeline, and read the files. ''' base_path = 'normal-basic' compare_path = 'add-basic' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path) # run with all images make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x', '12']) call_command('runpipeline', self.base_run) self.sources_base = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.relations_base = pd.read_parquet( os.path.join(self.base_run, 'relations.parquet')) # run with add image make_testdir(self.compare_run) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02']) call_command('runpipeline', self.compare_run) self.ass_backup = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) self.sources_backup = pd.read_parquet( os.path.join(self.compare_run, 'sources.parquet')) index = self.sources_backup.index self.sources_backup_db = pd.DataFrame( [Source.objects.get(id=ind).n_meas for ind in index], index=index, columns=['n_meas']) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x', '12']) call_command('runpipeline', self.compare_run) self.ass_compare = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) self.sources_compare = pd.read_parquet( os.path.join(self.compare_run, 'sources.parquet')) index = self.sources_compare.index self.sources_compare_db = pd.DataFrame( [Source.objects.get(id=ind).n_meas for ind in index], index=index, columns=['n_meas']) self.relations_compare = pd.read_parquet( os.path.join(self.compare_run, 'relations.parquet')) # remove directories shutil.rmtree(self.base_run) shutil.rmtree(self.compare_run)
def setUpTestData(self): ''' Set up directory to test data and run the pipeline. ''' base_path = 'normal-basic-forced' compare_path = 'add-parallel-basic-forced' self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path) self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path) # normal run make_testdir(self.base_run) gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.base_run) self.forced_base = {} for f in os.listdir(self.base_run): if f[:6] == 'forced': self.forced_base[f] = pd.read_parquet( os.path.join(self.base_run, f)) self.sources_norm = pd.read_parquet( os.path.join(self.base_run, 'sources.parquet')) self.ass_norm = pd.read_parquet( os.path.join(self.base_run, 'associations.parquet')) # add image run make_testdir(self.compare_run) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02']) call_command('runpipeline', self.compare_run) gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR, ['01', '03x', '02', '05x', '06x']) call_command('runpipeline', self.compare_run) self.forced_compare = {} for f in os.listdir(self.compare_run): if f[:6] == 'forced' and f[-3:] != 'bak': self.forced_compare[f] = pd.read_parquet( os.path.join(self.compare_run, f)) self.sources_compare = pd.read_parquet( os.path.join(self.compare_run, 'sources.parquet')) self.ass_compare = pd.read_parquet( os.path.join(self.compare_run, 'associations.parquet')) # remove test directories shutil.rmtree(self.base_run) shutil.rmtree(self.compare_run)