def setUpClass(cls):
print('Setup starting!')
### CREATE INSTANCE OF THE CLASS
# connection stuff
ssh_config_alias = input(
'Please enter the name in your .ssh/config file that you want to connect to BlueCrystal with (i.e. the host): '
)
cls.ssh_config_alias = ssh_config_alias
base_path_on_cluster = input(
'Please enter a path on the cluster to create unittest directories: '
)
if base_path_on_cluster == '':
base_path_on_cluster = '/projects/flex1/database/wcm_suite/unittest'
test_all_state_conversions = input(
'Would you like to test converting all the states from mat files to pandas dataframes? (True/False) NOTE: test all the states takes a while (10-15 mins) so don\'t worry if you\'re debugging something completely unrelated don\'t worry about it but it does need to be done every now and then (at least at the end of a change when you think you\'re done and ready to go). '
)
if test_all_state_conversions == '':
test_all_state_conversions = 'False'
username = input('Please enter your username on the cluster: ')
if username == '':
username = '******'
location_of_state_mat_file = input(
'Please enter the name and location of a state-?.mat file on the local computer (NOTE: This will not be touched as a copy will be made in a temporary directory): '
)
if location_of_state_mat_file == '':
location_of_state_mat_file = '/space/oc13378/myprojects/github/published_libraries/whole_cell_modelling_suite/whole_cell_modelling_suite/test_data/state-20.mat'
cls.relative_base_that_gets_deleted = 'wcms'
cls.test_all_state_conversions = eval(test_all_state_conversions)
cls.location_of_state_mat_file = location_of_state_mat_file
cls.base_path_on_cluster = base_path_on_cluster
cls.username = username
cls.forename = 'unit'
cls.surname = 'test'
cls.email = '*****@*****.**'
cls.output_path = 'output'
cls.full_output_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.output_path
cls.runfiles_path = 'runfiles'
cls.full_runfiles_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.runfiles_path
cls.out_and_error_files = 'out_and_error_files'
cls.full_out_and_error_files = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.out_and_error_files
cls.wholecell_model_master = 'WholeCell-master'
cls.affiliation = 'Test affiliation.'
bg_conn = connections.Karr2012Bg(cls.username,
ssh_config_alias,
cls.forename,
cls.surname,
cls.email,
cls.full_output_path,
cls.full_runfiles_path,
cls.wholecell_model_master,
affiliation=cls.affiliation)
cls.bg_conn = bg_conn
# submission stuff
# the fastest simulating 1ko is MG_107
cls.fast_ko = 'MG_107'
cls.submission_name = 'test_submisson'
cls.ko_name_to_set_dict = {
'wcms': (cls.fast_ko, ),
'test_sim2': (cls.fast_ko, )
}
cls.repetitions_of_a_unique_simulation = 1
cls.master_dir = '/projects/flex1/database/WholeCell-master'
cls.updateCentralDbFunctionName = 'updateCentralDbFunctionTest'
cls.convertDataFunctionName = 'convertDataFunctionTest'
cls.data_conversion_command_code = 'data_conversion_command_code_test'
cls.temp_storage_path = '/space/oc13378/myprojects/github/published_libraries/whole-cell-modelling-suite/whole-cell-modelling-suite/temp_storage'
cls.createDataDictForSpecialistFunctionsFunctionName = 'createDataDictForKos'
cls.createSubmissionScriptFunctionName = 'createWcmKoScript'
cls.createDictOfFileSourceToFileDestinationsFunctionName = 'createDictOfFileSourceToFileDestinationForKos'
cls.createAllFilesFunctionName = 'createAllFilesForKo'
cls.functionToGetReleventData = 'getGrowthAndDivisionTime'
cls.queue_name = 'cpu'
cls.queue_nameUT = 'cpu'
# CREATE SUBIMSSION INSTANCES
cls.bg_wcm_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bg_conn,
cls.ko_name_to_set_dict,
cls.queue_name,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionName,
cls.createDataDictForSpecialistFunctionsFunctionName,
cls.createSubmissionScriptFunctionName,
cls.createDictOfFileSourceToFileDestinationsFunctionName,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
# CREATE A UNITTEST SUBMISION INSTANCE
cls.createDataDictForSpecialistFunctionsFunctionNameUT = 'createDataDictForUnittest'
cls.createSubmissionScriptFunctionNameUT = 'createUnittestScript'
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT = 'createDictOfFileSourceToFileDestinationForUnittest'
cls.createAllFilesFunctionNameUT = 'createAllFilesForUnittest'
cls.getDataForDbFunctionName = 'getGrowthAndDivisionTime'
quick_list_of_states_to_convert = ['basic_summary']
long_list_of_states_to_convert = [
'basic_summary', 'mature_rna_counts', 'metabolic_reaction_fluxs',
'mature_protein_monomer_counts', 'mature_protein_complex_counts'
]
if cls.test_all_state_conversions == True:
list_of_states_to_convert = long_list_of_states_to_convert.copy()
else:
list_of_states_to_convert = quick_list_of_states_to_convert.copy()
cls.bg_unittest_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bg_conn,
cls.ko_name_to_set_dict,
cls.queue_nameUT,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionNameUT,
cls.createDataDictForSpecialistFunctionsFunctionNameUT,
cls.createSubmissionScriptFunctionNameUT,
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
# CREATE A JOB MANAGEMENT INSTANCE IN TEST MODE
cls.test_manager = job_management.SubmissionManagerKarr2012(
cls.bg_unittest_ko_sub,
cls.updateCentralDbFunctionName,
cls.convertDataFunctionName,
cls.getDataForDbFunctionName,
list_of_states_to_save=list_of_states_to_convert,
ko_db_path_relative_to_db_connection_flex1=
'database/ko_db/unittest_ko_db/unittest_ko.db',
test_mode=True)
### DEFINE LOCAL CLASS VARIABLES NEEDED THROUGHOUT TESTING
cls.base_dir = 'base_connection_test_directory'
cls.create_file_path = 'base_connection_test_directory/test_createLocalFile'
cls.sub_script_dir = 'base_connection_test_directory/submission_script_test'
cls.move_fname = 'base_connection_test_rsync_remote_transfer_file.txt'
cls.move_dir1 = 'base_connection_test_directory/test_createLocalFile/test_directory1'
cls.move_dir2 = 'base_connection_test_directory/test_createLocalFile/test_directory2'
cls.test_localShellCommand_path = 'base_connection_test_directory/test_localShellCommand'
cls.test_localShellCommand_file = 'localShellCommand.file'
cls.tmp_local_data_storage_path = 'base_connection_test_directory/local_data_storage'
### CREATE DIRS FILES ETC FIR TESTS
# check that the base connection test directoy doesn't already exist
if os.path.isdir(cls.base_dir):
raise ValueError(
'The directory base_connection_test_directory must not exist, please move some where else.'
)
# create directories needed for test
pathlib.Path(cls.sub_script_dir).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir1).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir2).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.test_localShellCommand_path).mkdir(parents=True,
exist_ok=True)
pathlib.Path(cls.tmp_local_data_storage_path).mkdir(parents=True,
exist_ok=True)
# create a file for transfer tests
list_of_lines_of_file = ['This', 'is', 'a', 'test']
list_of_files_to_make = [
cls.move_dir1 + "/" + cls.move_fname,
cls.test_localShellCommand_path + '/' +
cls.test_localShellCommand_file
]
for file_name in list_of_files_to_make:
with open(file_name, mode='wt', encoding='utf-8') as myfile:
for line in list_of_lines_of_file:
myfile.write(line + "\n")
### CREATE REMOTE DIRECTORY THAT GETS DELETED EVERYTIME
mkdir_cmd = [
'mkdir -p ' + cls.base_path_on_cluster + '/' +
cls.relative_base_that_gets_deleted
]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand, mkdir_cmd)
if raw_out['return_code'] != 0:
print(
"Warning, could not create the remote dir. This currently shouldn't cause an error but may do later on. raw_out = ",
raw_out)
# make remote dirs
cls.tmp_mat_files = 'example_simulation'
cls.full_path_to_sim_example = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.tmp_mat_files
mkdir_cmd = ['mkdir -p ' + cls.full_path_to_sim_example]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand, mkdir_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
"Whilst constructing the class we couldn't create remote directories!. raw_out = ",
raw_out)
# also add a whole simulation there
source_path = '/projects/flex1/database/mat_files/version73_example_mats_files/1/' # put the trailing slash so that it only copies the contents of the dir
transfer_cmd = [
'rsync -aP ' + source_path + ' ' + cls.full_path_to_sim_example
]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand,
transfer_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
"Could not transfer example simulation data to the temporary directory! raw_out = ",
raw_out)
print('Setup up finishing!')
def setUpClass(cls):
### CREATE INSTANCE OF THE CLASS
# connection stuff
ssh_config_alias = input(
'Please enter the name in your .ssh/config file that you want to connect to BlueCrystal Phase III with (i.e. the host): '
)
cls.ssh_config_alias = ssh_config_alias
base_path_on_cluster = input(
'Please enter a path on the cluster to create unittest directories: '
)
if base_path_on_cluster == '':
base_path_on_cluster = '/panfs/panasas01/bluegem-flex1/database/wcm_suite/unittest'
username = input('Please enter your username on the cluster: ')
if username == '':
username = '******'
cls.base_path_on_cluster = base_path_on_cluster
cls.relative_base_that_gets_deleted = 'wcms'
cls.username = username
cls.forename = 'unit'
cls.surname = 'test'
cls.email = '*****@*****.**'
cls.output_path = 'output'
cls.full_output_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.output_path
cls.runfiles_path = 'runfiles'
cls.full_runfiles_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.runfiles_path
cls.out_and_error_files = 'out_and_error_files'
cls.full_out_and_error_files = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.out_and_error_files
cls.wholecell_model_master = 'WholeCell-master'
cls.affiliation = 'Test affiliation.'
bc3_conn = connections.Karr2012Bc3(cls.username,
ssh_config_alias,
cls.forename,
cls.surname,
cls.email,
cls.full_output_path,
cls.full_runfiles_path,
cls.wholecell_model_master,
affiliation=cls.affiliation)
cls.bc3_conn = bc3_conn
# submission stuff
# the fastest simulating 1ko is MG_107
cls.fast_ko = 'MG_107'
cls.submission_name = 'test_submisson'
cls.ko_name_to_set_dict = {
'wcms': (cls.fast_ko, ),
'test_sim2': (cls.fast_ko, )
}
cls.repetitions_of_a_unique_simulation = 1
cls.master_dir = '/panfs/panasas01/emat/oc13378/WholeCell/wc/mg/WholeCell-master'
cls.updateCentralDbFunctionName = 'updateCentralDbFunctionTest'
cls.convertDataFunctionName = 'convertDataFunctionTest'
cls.getDataForDbFunctionName = 'getGrowthAndDivisionTime'
cls.temp_storage_path = '/space/oc13378/myprojects/github/published_libraries/whole-cell-modelling-suite/whole-cell-modelling-suite/temp_storage'
cls.createDataDictForSpecialistFunctionsFunctionName = 'createDataDictForKos'
cls.createSubmissionScriptFunctionName = 'createWcmKoScript'
cls.createDictOfFileSourceToFileDestinationsFunctionName = 'createDictOfFileSourceToFileDestinationForKos'
cls.createAllFilesFunctionName = 'createAllFilesForKo'
cls.queue_name = 'short'
cls.queue_nameUT = 'testq'
# CREATE SUBIMSSION INSTANCES
cls.bc3_wcm_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bc3_conn,
cls.ko_name_to_set_dict,
cls.queue_name,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionName,
cls.createDataDictForSpecialistFunctionsFunctionName,
cls.createSubmissionScriptFunctionName,
cls.createDictOfFileSourceToFileDestinationsFunctionName,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
# CREATE A UNITTEST SUBMISION INSTANCE
cls.createDataDictForSpecialistFunctionsFunctionNameUT = 'createDataDictForUnittest'
cls.createSubmissionScriptFunctionNameUT = 'createUnittestScript'
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT = 'createDictOfFileSourceToFileDestinationForUnittest'
cls.createAllFilesFunctionNameUT = 'createAllFilesForUnittest'
quick_list_of_states_to_convert = ['basic_summary']
long_list_of_states_to_convert = [
'basic_summary', 'mature_rna_counts', 'metabolic_reaction_fluxs',
'mature_protein_monomer_counts', 'mature_protein_complex_counts'
]
cls.bc3_unittest_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bc3_conn,
cls.ko_name_to_set_dict,
cls.queue_nameUT,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionNameUT,
cls.createDataDictForSpecialistFunctionsFunctionNameUT,
cls.createSubmissionScriptFunctionNameUT,
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
### DEFINE LOCAL CLASS VARIABLES NEEDED THROUGHOUT TESTING
cls.base_dir = 'base_connection_test_directory'
cls.create_file_path = 'base_connection_test_directory/test_createLocalFile'
cls.sub_script_dir = 'base_connection_test_directory/submission_script_test'
cls.move_fname = 'base_connection_test_rsync_remote_transfer_file.txt'
cls.move_dir1 = 'base_connection_test_directory/test_createLocalFile/test_directory1'
cls.move_dir2 = 'base_connection_test_directory/test_createLocalFile/test_directory2'
cls.test_localShellCommand_path = 'base_connection_test_directory/test_localShellCommand'
cls.test_localShellCommand_file = 'localShellCommand.file'
cls.tmp_local_data_storage_path = 'base_connection_test_directory/local_data_storage'
### CREATE DIRS FILES ETC FIR TESTS
# check that the base connection test directoy doesn't already exist
if os.path.isdir(cls.base_dir):
raise ValueError(
'The directory base_connection_test_directory must not exist, please move some where else.'
)
# create directories needed for test
pathlib.Path(cls.sub_script_dir).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir1).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir2).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.test_localShellCommand_path).mkdir(parents=True,
exist_ok=True)
pathlib.Path(cls.tmp_local_data_storage_path).mkdir(parents=True,
exist_ok=True)
# create a copy of the state-?.mat file in the temp directory
cls.bc3_conn.transferFile(cls.location_of_state_mat_file,
cls.tmp_local_data_storage_path,
remote=False)
# create a file for transfer tests
list_of_lines_of_file = ['This', 'is', 'a', 'test']
list_of_files_to_make = [
cls.move_dir1 + "/" + cls.move_fname,
cls.test_localShellCommand_path + '/' +
cls.test_localShellCommand_file
]
for file_name in list_of_files_to_make:
with open(file_name, mode='wt', encoding='utf-8') as myfile:
for line in list_of_lines_of_file:
myfile.write(line + "\n")