def setUpTestData(self):
'''
Set up directory to test data and run the pipeline.
'''
base_path = 'epoch-add-parallel-deruiter-forced'
self.base_run = os.path.join(
s.PIPELINE_WORKING_DIR, base_path
)
# add image run
make_testdir(self.base_run)
gen_config.gen_config(
base_path,
s.PIPELINE_WORKING_DIR,
['01', '03x', '02']
)
call_command('runpipeline', self.base_run)
gen_config.gen_config(
base_path,
s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x', '12']
)
call_command('runpipeline', self.base_run)
self.forced = {}
for f in os.listdir(self.base_run):
if f[:6] == 'forced' and f[-3:] != 'bak':
self.forced[f] = pd.read_parquet(
os.path.join(self.base_run, f)
)
# remove directory
shutil.rmtree(self.base_run)
def setUpTestData(self):
'''
Set up directories to test data, run the pipeline, and read files.
'''
base_path = 'epoch-add-parallel-deruiter'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
# run with add image
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02'])
call_command('runpipeline', self.base_run)
self.ass_backup = pd.read_parquet(
os.path.join(self.base_run, 'associations.parquet'))
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.base_run)
self.sources = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations = pd.read_parquet(
os.path.join(self.base_run, 'relations.parquet'))
self.ass = pd.read_parquet(
os.path.join(self.base_run, 'associations.parquet'))
# remove test directory
shutil.rmtree(self.base_run)
def setUpTestData(self):
'''
Set up directories to test data, run the pipeline, and read files.
'''
base_path = 'normal-advanced'
compare_path = 'parallel-advanced'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path)
# run with normal
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.base_run)
self.sources_base = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations_base = pd.read_parquet(
os.path.join(self.base_run, 'relations.parquet'))
# run with parallel
make_testdir(self.compare_run)
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.compare_run)
self.sources_compare = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations_compare = pd.read_parquet(
os.path.join(self.compare_run, 'relations.parquet'))
# remove directories
shutil.rmtree(self.base_run)
shutil.rmtree(self.compare_run)
def setUpTestData(self):
'''
Set up directories to test data, run the pipeline, and read the files.
'''
base_path = 'normal-basic'
compare_path = 'add-basic'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path)
# run with all images
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x', '12'])
call_command('runpipeline', self.base_run)
self.sources_base = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations_base = pd.read_parquet(
os.path.join(self.base_run, 'relations.parquet'))
# run with add image
make_testdir(self.compare_run)
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02'])
call_command('runpipeline', self.compare_run)
self.ass_backup = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
self.sources_backup = pd.read_parquet(
os.path.join(self.compare_run, 'sources.parquet'))
index = self.sources_backup.index
self.sources_backup_db = pd.DataFrame(
[Source.objects.get(id=ind).n_meas for ind in index],
index=index,
columns=['n_meas'])
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x', '12'])
call_command('runpipeline', self.compare_run)
self.ass_compare = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
self.sources_compare = pd.read_parquet(
os.path.join(self.compare_run, 'sources.parquet'))
index = self.sources_compare.index
self.sources_compare_db = pd.DataFrame(
[Source.objects.get(id=ind).n_meas for ind in index],
index=index,
columns=['n_meas'])
self.relations_compare = pd.read_parquet(
os.path.join(self.compare_run, 'relations.parquet'))
# remove directories
shutil.rmtree(self.base_run)
shutil.rmtree(self.compare_run)
def setUpTestData(self):
'''
Set up directories to test data, run the pipeline, and read the files.
'''
base_path = 'normal-basic'
compare_path = 'add-add-parallel-basic'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path)
# run with all images
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x', '12'])
call_command('runpipeline', self.base_run)
self.sources_base = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations_base = pd.read_parquet(
os.path.join(self.base_run, 'relations.parquet'))
# run with add image
make_testdir(self.compare_run)
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02'])
call_command('runpipeline', self.compare_run)
self.ass_backup = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
# run with epoch 05 06
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.compare_run)
self.ass_backup_mid = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
# run with epoch 12
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x', '12'])
call_command('runpipeline', self.compare_run)
self.ass_compare = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
self.sources_compare = pd.read_parquet(
os.path.join(self.compare_run, 'sources.parquet'))
self.relations_compare = pd.read_parquet(
os.path.join(self.compare_run, 'relations.parquet'))
# remove test directories
shutil.rmtree(self.base_run)
shutil.rmtree(self.compare_run)
def setUpTestData(self):
'''
Set up directory to test data and run the pipeline.
'''
base_path = 'normal-basic-forced'
compare_path = 'add-parallel-basic-forced'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
self.compare_run = os.path.join(s.PIPELINE_WORKING_DIR, compare_path)
# normal run
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.base_run)
self.forced_base = {}
for f in os.listdir(self.base_run):
if f[:6] == 'forced':
self.forced_base[f] = pd.read_parquet(
os.path.join(self.base_run, f))
self.sources_norm = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.ass_norm = pd.read_parquet(
os.path.join(self.base_run, 'associations.parquet'))
# add image run
make_testdir(self.compare_run)
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02'])
call_command('runpipeline', self.compare_run)
gen_config.gen_config(compare_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.compare_run)
self.forced_compare = {}
for f in os.listdir(self.compare_run):
if f[:6] == 'forced' and f[-3:] != 'bak':
self.forced_compare[f] = pd.read_parquet(
os.path.join(self.compare_run, f))
self.sources_compare = pd.read_parquet(
os.path.join(self.compare_run, 'sources.parquet'))
self.ass_compare = pd.read_parquet(
os.path.join(self.compare_run, 'associations.parquet'))
# remove test directories
shutil.rmtree(self.base_run)
shutil.rmtree(self.compare_run)
def setUpTestData(self):
'''
Set up directory to test data and run the pipeline.
'''
base_path = 'epoch-advanced'
self.base_run = os.path.join(s.PIPELINE_WORKING_DIR, base_path)
# setup test directory
make_testdir(self.base_run)
gen_config.gen_config(base_path, s.PIPELINE_WORKING_DIR,
['01', '03x', '02', '05x', '06x'])
call_command('runpipeline', self.base_run)
# read output
self.sources = pd.read_parquet(
os.path.join(self.base_run, 'sources.parquet'))
self.relations = pd.read_parquet(
os.path.join(self.base_run, 'relations.parquet'))
# remove test directory
shutil.rmtree(self.base_run)
