class TestEload(TestCase):
resources_folder = os.path.join(ROOT_DIR, 'tests', 'resources')
def setUp(self):
config_file = os.path.join(self.resources_folder, 'submission_config.yml')
load_config(config_file)
# Need to set the directory so that the relative path set in the config file works from the top directory
os.chdir(ROOT_DIR)
self.eload = Eload(55)
self.original_config = EloadConfig(os.path.join(self.eload.eload_dir, 'original_config.yml'))
self.updated_config = EloadConfig(os.path.join(self.eload.eload_dir, 'updated_config.yml'))
# Setup the config
self.eload.eload_cfg.content = deepcopy(self.original_config.content)
self.original_updated_cfg = deepcopy(self.updated_config.content)
self.updated_config.set('version', value=__version__)
# Get the log file name
self.logfile_name = os.path.join(self.eload.eload_dir, str(self.eload.eload) + "_submission.log")
def tearDown(self):
self.updated_config.content = self.original_updated_cfg
# remove the config its backup and the log file
for file_path in [self.eload.eload_cfg.config_file, f'{self.eload.eload_cfg.config_file}.1', self.logfile_name]:
if os.path.exists(file_path):
os.remove(file_path)
def test_create_log_file(self):
# Creating a second eload object to test whether the logging file handler
# has been created twice
eload2 = Eload(self.eload.eload_num)
self.eload.info("Testing the creation of logging file")
assert os.path.exists(self.logfile_name)
with open(self.logfile_name, "r") as test_logfile:
k = [i for i in test_logfile.readlines() if "Testing the creation of logging file" in i]
# Checking if the log message is written only once in the log file
assert len(k) == 1
def test_upgrade_config(self):
"""Tests config upgrade for a post-brokering config."""
self.eload.upgrade_config_if_needed('analysis alias')
self.assertEqual(self.updated_config.content, self.eload.eload_cfg.content)
def test_upgrade_config_no_analysis_alias_passed(self):
"""If no analysis alias is passed, it should retrieve from metadata when possible."""
with patch('eva_submission.config_migration.EvaXlsxReader') as mock_reader:
mock_reader.return_value.analysis = [{'Analysis Alias': 'analysis alias'}]
self.eload.upgrade_config_if_needed(analysis_alias=None)
self.assertEqual(self.updated_config.content, self.eload.eload_cfg.content)
def test_upgrade_config_already_updated(self):
"""An already up-to-date config shouldn't get modified."""
original_content = deepcopy(self.updated_config.content)
self.eload.upgrade_config_if_needed('analysis alias')
self.assertEqual(original_content, self.updated_config.content)
def setUp(self):
config_file = os.path.join(self.resources_folder, 'submission_config.yml')
load_config(config_file)
# Need to set the directory so that the relative path set in the config file works from the top directory
os.chdir(ROOT_DIR)
self.eload = Eload(55)
self.updated_config = EloadConfig(os.path.join(self.eload.eload_dir, 'updated_config.yml'))
# Used to restore test config after each test
self.original_cfg = deepcopy(self.eload.eload_cfg.content)
self.original_updated_cfg = deepcopy(self.updated_config.content)
self.updated_config.set('version', value=__version__)
def __init__(self, eload_number: int):
self.eload_num = eload_number
self.eload = f'ELOAD_{eload_number}'
self.eload_dir = os.path.abspath(
os.path.join(cfg['eloads_dir'], self.eload))
self.eload_cfg = EloadConfig(
os.path.join(self.eload_dir, '.' + self.eload + '_config.yml'))
os.makedirs(self.eload_dir, exist_ok=True)
for k in directory_structure:
os.makedirs(self._get_dir(k), exist_ok=True)
def setUp(self):
config_file = os.path.join(self.resources_folder, 'submission_config.yml')
load_config(config_file)
# Need to set the directory so that the relative path set in the config file works from the top directory
os.chdir(ROOT_DIR)
self.eload = Eload(55)
self.original_config = EloadConfig(os.path.join(self.eload.eload_dir, 'original_config.yml'))
self.updated_config = EloadConfig(os.path.join(self.eload.eload_dir, 'updated_config.yml'))
# Setup the config
self.eload.eload_cfg.content = deepcopy(self.original_config.content)
self.original_updated_cfg = deepcopy(self.updated_config.content)
self.updated_config.set('version', value=__version__)
# Get the log file name
self.logfile_name = os.path.join(self.eload.eload_dir, str(self.eload.eload) + "_submission.log")
def __init__(self, eload_number: int, config_object: EloadConfig = None):
self.eload_num = eload_number
self.eload = f'ELOAD_{eload_number}'
self.eload_dir = os.path.abspath(
os.path.join(cfg['eloads_dir'], self.eload))
if config_object:
self.eload_cfg = config_object
else:
self.eload_cfg = EloadConfig(
os.path.join(self.eload_dir, '.' + self.eload + '_config.yml'))
os.makedirs(self.eload_dir, exist_ok=True)
for k in directory_structure:
os.makedirs(self._get_dir(k), exist_ok=True)
self.create_log_file()
def test_remove_from_config(self):
eload_cfg = EloadConfig()
eload_cfg.set('level1', 'level2', 'level3', value='value')
assert eload_cfg.pop('level1', 'lunch time',
default='spaghetti') == 'spaghetti'
assert eload_cfg.pop('level1', 'level2', 'level3',
default='spaghetti') == 'value'
assert eload_cfg.pop('level1', 'level2', 'level3',
default='spaghetti') == 'spaghetti'
def test_add_to_config(self):
eload_cfg = EloadConfig()
eload_cfg.set('key', value='Value1')
assert eload_cfg.content['key'] == 'Value1'
eload_cfg.set('level1', 'level2', 'level3', value='Value2')
assert eload_cfg.content['level1']['level2']['level3'] == 'Value2'
class TestEload(TestCase):
resources_folder = os.path.join(ROOT_DIR, 'tests', 'resources')
def setUp(self):
config_file = os.path.join(self.resources_folder, 'submission_config.yml')
load_config(config_file)
# Need to set the directory so that the relative path set in the config file works from the top directory
os.chdir(ROOT_DIR)
self.eload = Eload(55)
self.updated_config = EloadConfig(os.path.join(self.eload.eload_dir, 'updated_config.yml'))
# Used to restore test config after each test
self.original_cfg = deepcopy(self.eload.eload_cfg.content)
self.original_updated_cfg = deepcopy(self.updated_config.content)
self.updated_config.set('version', value=__version__)
def tearDown(self):
self.eload.eload_cfg.content = self.original_cfg
self.updated_config.content = self.original_updated_cfg
if os.path.exists(f'{self.eload.eload_cfg.config_file}.old'):
os.remove(f'{self.eload.eload_cfg.config_file}.old')
def test_upgrade_config(self):
"""Tests config upgrade for a post-brokering config."""
self.eload.upgrade_config_if_needed('analysis alias')
self.assertEqual(self.updated_config.content, self.eload.eload_cfg.content)
def test_upgrade_config_no_analysis_alias_passed(self):
"""If no analysis alias is passed, it should retrieve from metadata when possible."""
with patch('eva_submission.config_migration.EvaXlsxReader') as mock_reader:
mock_reader.return_value.analysis = [{'Analysis Alias': 'analysis alias'}]
self.eload.upgrade_config_if_needed(analysis_alias=None)
self.assertEqual(self.updated_config.content, self.eload.eload_cfg.content)
def test_upgrade_config_already_updated(self):
"""An already up-to-date config shouldn't get modified."""
original_content = deepcopy(self.updated_config.content)
self.eload.upgrade_config_if_needed('analysis alias')
self.assertEqual(original_content, self.updated_config.content)
def setUp(self) -> None:
self.eload_cfg = EloadConfig()
self.eload_cfg.load_config_file(
os.path.join(self.resources_folder, 'testconfig.yml'))
class TestEloadConfig(TestCase):
resources_folder = os.path.join(ROOT_DIR, 'tests', 'resources')
def setUp(self) -> None:
self.eload_cfg = EloadConfig()
self.eload_cfg.load_config_file(
os.path.join(self.resources_folder, 'testconfig.yml'))
def tearDown(self) -> None:
self.eload_cfg.clear()
for i in range(5):
if os.path.exists(self.eload_cfg.config_file + '.' + str(i)):
os.remove(self.eload_cfg.config_file + '.' + str(i))
def test_add_to_config(self):
self.eload_cfg.set('key', value='Value1')
assert self.eload_cfg.content['key'] == 'Value1'
self.eload_cfg.set('level1', 'level2', 'level3', value='Value2')
assert self.eload_cfg.content['level1']['level2']['level3'] == 'Value2'
def test_remove_from_config(self):
self.eload_cfg.set('level1', 'level2', 'level3', value='value')
assert self.eload_cfg.pop('level1', 'lunch time',
default='spaghetti') == 'spaghetti'
assert self.eload_cfg.pop('level1',
'level2',
'level3',
default='spaghetti') == 'value'
assert self.eload_cfg.pop('level1',
'level2',
'level3',
default='spaghetti') == 'spaghetti'
def test_backup(self):
touch(self.eload_cfg.config_file)
for i in range(4):
touch(self.eload_cfg.config_file + '.' + str(i))
self.eload_cfg.backup()
assert not os.path.exists(self.eload_cfg.config_file)
for i in range(5):
assert os.path.exists(self.eload_cfg.config_file + '.' + str(i))
class Eload(AppLogger):
def __init__(self, eload_number: int, config_object: EloadConfig = None):
self.eload_num = eload_number
self.eload = f'ELOAD_{eload_number}'
self.eload_dir = os.path.abspath(
os.path.join(cfg['eloads_dir'], self.eload))
if config_object:
self.eload_cfg = config_object
else:
self.eload_cfg = EloadConfig(
os.path.join(self.eload_dir, '.' + self.eload + '_config.yml'))
os.makedirs(self.eload_dir, exist_ok=True)
for k in directory_structure:
os.makedirs(self._get_dir(k), exist_ok=True)
self.create_log_file()
@property
def metadata_connection_handle(self):
return get_metadata_connection_handle(cfg['maven']['environment'],
cfg['maven']['settings_file'])
def create_nextflow_temp_output_directory(self, base=None):
random_string = ''.join(
random.choice(string.ascii_letters) for i in range(6))
if base is None:
output_dir = os.path.join(self.eload_dir,
'nextflow_output_' + random_string)
else:
output_dir = os.path.join(base, 'nextflow_output_' + random_string)
os.makedirs(output_dir)
return output_dir
def _get_dir(self, key):
return os.path.join(self.eload_dir, directory_structure[key])
@cached_property
def now(self):
return datetime.now()
def create_log_file(self):
logfile_name = os.path.join(self.eload_dir,
str(self.eload) + "_submission.log")
if logfile_name not in eload_logging_files:
log_cfg.add_file_handler(logfile_name)
eload_logging_files.add(logfile_name)
def upgrade_config_if_needed(self, analysis_alias=None):
"""
Upgrades configs to the current version, making a backup first and using the provided analysis alias for all
vcf files. Currently doesn't perform any other version upgrades.
"""
if 'version' not in self.eload_cfg:
self.debug(
f'No version found in config, upgrading to version {__version__}.'
)
self.eload_cfg.backup()
upgrade_version_0_1(self.eload_cfg, analysis_alias)
else:
self.debug(
f"Config is version {self.eload_cfg.query('version')}, not upgrading."
)
def update_config_with_hold_date(self,
project_accession,
project_alias=None):
hold_date = get_hold_date_from_ena(project_accession, project_alias)
self.eload_cfg.set('brokering', 'ena', 'hold_date', value=hold_date)
def update_metadata_from_config(self,
input_spreadsheet,
output_spreadsheet=None):
reader = EvaXlsxReader(input_spreadsheet)
single_analysis_alias = None
if len(reader.analysis) == 1:
single_analysis_alias = reader.analysis[0].get('Analysis Alias')
sample_rows = []
for sample_row in reader.samples:
if self.eload_cfg.query('brokering', 'Biosamples', 'Samples',
sample_row.get('Sample Name')):
sample_rows.append({
'row_num':
sample_row.get('row_num'),
'Analysis Alias':
sample_row.get('Analysis Alias') or single_analysis_alias,
'Sample ID':
sample_row.get('Sample Name'),
'Sample Accession':
self.eload_cfg['brokering']['Biosamples']['Samples'][
sample_row.get('Sample Name')]
})
else:
sample_rows.append(sample_row)
file_rows = []
analyses = self.eload_cfg['brokering']['analyses']
for analysis in analyses:
for vcf_file_name in analyses[analysis]['vcf_files']:
vcf_file_info = self.eload_cfg['brokering']['analyses'][
analysis]['vcf_files'][vcf_file_name]
# Add the vcf file
file_rows.append({
'Analysis Alias':
analysis,
'File Name':
self.eload + '/' + os.path.basename(vcf_file_name),
'File Type':
'vcf',
'MD5':
vcf_file_info['md5']
})
# Add the index file
if vcf_file_info['index'].endswith('.csi'):
file_type = 'csi'
else:
file_type = 'tabix'
file_rows.append({
'Analysis Alias':
analysis,
'File Name':
self.eload + '/' +
os.path.basename(vcf_file_info['index']),
'File Type':
file_type,
'MD5':
vcf_file_info['index_md5']
})
project_row = reader.project
if self.eload_cfg.query('brokering', 'ena', 'existing_project'):
project_row['Project Alias'] = self.eload_cfg.query(
'brokering', 'ena', 'PROJECT')
if output_spreadsheet:
eva_xls_writer = EvaXlsxWriter(input_spreadsheet,
output_spreadsheet)
else:
eva_xls_writer = EvaXlsxWriter(input_spreadsheet)
eva_xls_writer.set_project(project_row)
eva_xls_writer.set_samples(sample_rows)
eva_xls_writer.set_files(file_rows)
eva_xls_writer.save()
return output_spreadsheet
@staticmethod
def _check_pass_or_fail(check_dict):
if check_dict and check_dict.get('forced'):
return 'FORCED'
if check_dict and check_dict.get('pass'):
return 'PASS'
return 'FAIL'