def test_monitorSubmission(self):
print('test_monitorSubmission')
clear_db_cmd = [
'cd /panfs/panasas01/bluegem-flex1/database/ko_db/unittest_ko_db; rm unittest_ko.db; sqlite3 unittest_ko.db < unittest_ko_db.schema'
]
raw_out = self.bc3_conn.checkSuccess(self.bc3_conn.sendCommand,
clear_db_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
'Could not create a fresh unittest database". raw_out = ',
raw_out)
gene_code_to_id_dict = self.getDictOfJr358Codes(self.bc3_conn)
test_manager = job_management.SubmissionManagerKarr2012(
self.bc3_wcm_ko_sub,
gene_code_to_id_dict,
self.convertDataFunctionName,
self.updateCentralDbFunctionName,
self.getDataForDbFunctionName,
ko_db_path_relative_to_db_connection_flex1=
'database/ko_db/unittest_ko_db/unittest_ko.db',
test_mode=False)
test_db_cmd = [
'sqlite3 /panfs/panasas01/bluegem-flex1/database/ko_db/unittest_ko_db/unittest_ko.db "select count(simulationDetails.average_growth_rate) from simulationDetails join koIndex on simulationDetails.ko_index_id = koIndex.id join batchDescription on simulationDetails.batch_description_id = batchDescription.id join people on people.id = batchDescription.simulation_initiator_id"'
]
raw_out = self.bc3_conn.checkSuccess(self.bc3_conn.sendCommand,
test_db_cmd)
raw_out_correct = {'stderr': None, 'return_code': 0, 'stdout': '6\n'}
print('raw_out_correct = ', raw_out_correct)
print('raw_out = ', raw_out)
self.assertTrue(
(type(test_manager) is job_management.SubmissionManagerKarr2012)
and (raw_out_correct == raw_out))
def test_monitorSubmission(self):
print('test_monitorSubmission')
clear_db_cmd = [
'cd /projects/flex1/database/ko_db/unittest_ko_db; rm unittest_ko.db; sqlite3 unittest_ko.db < unittest_ko_db.schema'
]
raw_out = self.bg_conn.checkSuccess(self.bg_conn.sendCommand,
clear_db_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
'Could not create a fresh unittest database". raw_out = ',
raw_out)
gene_code_to_id_dict = self.getDictOfJr358Codes(self.bg_conn)
test_manager = job_management.SubmissionManagerKarr2012(
self.bg_unittest_ko_sub,
gene_code_to_id_dict,
self.convertDataFunctionName,
self.updateCentralDbFunctionName,
self.getDataForDbFunctionName,
ko_db_path_relative_to_db_connection_flex1=
'database/ko_db/unittest_ko_db/unittest_ko.db',
test_mode=False)
test_db_cmd = [
"echo $'.headers on\n.mode columns\nselect simulationDetails.time_when_pinchedDiameter_is_first_zero, simulationDetails.average_growth_rate, koIndex.number_of_kos, batchDescription.name, people.last_name, people.user_name from simulationDetails join koIndex on simulationDetails.ko_index_id = koIndex.id join batchDescription on simulationDetails.batch_description_id = batchDescription.id join people on people.id = batchDescription.simulation_initiator_id;' > tmp_sql.sqlite; sqlite3 /projects/flex1/database/ko_db/unittest_ko_db/unittest_ko.db < tmp_sql.sqlite; rm tmp_sql.sqlite"
]
raw_out = self.bg_conn.checkSuccess(self.bg_conn.sendCommand,
test_db_cmd)
raw_out_correct = {
'stderr':
None,
'return_code':
0,
'stdout':
'time_when_pinchedDiameter_is_first_zero average_growth_rate number_of_kos name last_name user_name \n--------------------------------------- ------------------- ------------- -------------- ---------- ----------\n0 1.68143350439e-05 1 test_submisson test oc13378 \n0 1.7248216697e-05 1 test_submisson test oc13378 \n0 1.5040890086e-05 1 test_submisson test oc13378 \n0 1.68143350439e-05 1 test_submisson test oc13378 \n0 1.7248216697e-05 1 test_submisson test oc13378 \n0 1.5040890086e-05 1 test_submisson test oc13378 \n'
}
print('raw_out_correct = ', raw_out_correct)
print('raw_out = ', raw_out)
self.assertTrue(
(type(test_manager) is job_management.SubmissionManagerKarr2012)
and (raw_out_correct == raw_out))
def setUpClass(cls):
print('Setup starting!')
### CREATE INSTANCE OF THE CLASS
# connection stuff
ssh_config_alias = input(
'Please enter the name in your .ssh/config file that you want to connect to BlueCrystal with (i.e. the host): '
)
cls.ssh_config_alias = ssh_config_alias
base_path_on_cluster = input(
'Please enter a path on the cluster to create unittest directories: '
)
if base_path_on_cluster == '':
base_path_on_cluster = '/projects/flex1/database/wcm_suite/unittest'
test_all_state_conversions = input(
'Would you like to test converting all the states from mat files to pandas dataframes? (True/False) NOTE: test all the states takes a while (10-15 mins) so don\'t worry if you\'re debugging something completely unrelated don\'t worry about it but it does need to be done every now and then (at least at the end of a change when you think you\'re done and ready to go). '
)
if test_all_state_conversions == '':
test_all_state_conversions = 'False'
username = input('Please enter your username on the cluster: ')
if username == '':
username = '******'
location_of_state_mat_file = input(
'Please enter the name and location of a state-?.mat file on the local computer (NOTE: This will not be touched as a copy will be made in a temporary directory): '
)
if location_of_state_mat_file == '':
location_of_state_mat_file = '/space/oc13378/myprojects/github/published_libraries/whole_cell_modelling_suite/whole_cell_modelling_suite/test_data/state-20.mat'
cls.relative_base_that_gets_deleted = 'wcms'
cls.test_all_state_conversions = eval(test_all_state_conversions)
cls.location_of_state_mat_file = location_of_state_mat_file
cls.base_path_on_cluster = base_path_on_cluster
cls.username = username
cls.forename = 'unit'
cls.surname = 'test'
cls.email = '*****@*****.**'
cls.output_path = 'output'
cls.full_output_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.output_path
cls.runfiles_path = 'runfiles'
cls.full_runfiles_path = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.runfiles_path
cls.out_and_error_files = 'out_and_error_files'
cls.full_out_and_error_files = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.out_and_error_files
cls.wholecell_model_master = 'WholeCell-master'
cls.affiliation = 'Test affiliation.'
bg_conn = connections.Karr2012Bg(cls.username,
ssh_config_alias,
cls.forename,
cls.surname,
cls.email,
cls.full_output_path,
cls.full_runfiles_path,
cls.wholecell_model_master,
affiliation=cls.affiliation)
cls.bg_conn = bg_conn
# submission stuff
# the fastest simulating 1ko is MG_107
cls.fast_ko = 'MG_107'
cls.submission_name = 'test_submisson'
cls.ko_name_to_set_dict = {
'wcms': (cls.fast_ko, ),
'test_sim2': (cls.fast_ko, )
}
cls.repetitions_of_a_unique_simulation = 1
cls.master_dir = '/projects/flex1/database/WholeCell-master'
cls.updateCentralDbFunctionName = 'updateCentralDbFunctionTest'
cls.convertDataFunctionName = 'convertDataFunctionTest'
cls.data_conversion_command_code = 'data_conversion_command_code_test'
cls.temp_storage_path = '/space/oc13378/myprojects/github/published_libraries/whole-cell-modelling-suite/whole-cell-modelling-suite/temp_storage'
cls.createDataDictForSpecialistFunctionsFunctionName = 'createDataDictForKos'
cls.createSubmissionScriptFunctionName = 'createWcmKoScript'
cls.createDictOfFileSourceToFileDestinationsFunctionName = 'createDictOfFileSourceToFileDestinationForKos'
cls.createAllFilesFunctionName = 'createAllFilesForKo'
cls.functionToGetReleventData = 'getGrowthAndDivisionTime'
cls.queue_name = 'cpu'
cls.queue_nameUT = 'cpu'
# CREATE SUBIMSSION INSTANCES
cls.bg_wcm_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bg_conn,
cls.ko_name_to_set_dict,
cls.queue_name,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionName,
cls.createDataDictForSpecialistFunctionsFunctionName,
cls.createSubmissionScriptFunctionName,
cls.createDictOfFileSourceToFileDestinationsFunctionName,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
# CREATE A UNITTEST SUBMISION INSTANCE
cls.createDataDictForSpecialistFunctionsFunctionNameUT = 'createDataDictForUnittest'
cls.createSubmissionScriptFunctionNameUT = 'createUnittestScript'
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT = 'createDictOfFileSourceToFileDestinationForUnittest'
cls.createAllFilesFunctionNameUT = 'createAllFilesForUnittest'
cls.getDataForDbFunctionName = 'getGrowthAndDivisionTime'
quick_list_of_states_to_convert = ['basic_summary']
long_list_of_states_to_convert = [
'basic_summary', 'mature_rna_counts', 'metabolic_reaction_fluxs',
'mature_protein_monomer_counts', 'mature_protein_complex_counts'
]
if cls.test_all_state_conversions == True:
list_of_states_to_convert = long_list_of_states_to_convert.copy()
else:
list_of_states_to_convert = quick_list_of_states_to_convert.copy()
cls.bg_unittest_ko_sub = job_management.SubmissionKarr2012(
cls.submission_name,
cls.bg_conn,
cls.ko_name_to_set_dict,
cls.queue_nameUT,
cls.full_output_path,
cls.full_runfiles_path,
cls.full_out_and_error_files,
cls.full_out_and_error_files,
cls.master_dir,
1,
cls.createAllFilesFunctionNameUT,
cls.createDataDictForSpecialistFunctionsFunctionNameUT,
cls.createSubmissionScriptFunctionNameUT,
cls.createDictOfFileSourceToFileDestinationsFunctionNameUT,
first_wait_time=1,
second_wait_time=1,
temp_storage_path=cls.temp_storage_path)
# CREATE A JOB MANAGEMENT INSTANCE IN TEST MODE
cls.test_manager = job_management.SubmissionManagerKarr2012(
cls.bg_unittest_ko_sub,
cls.updateCentralDbFunctionName,
cls.convertDataFunctionName,
cls.getDataForDbFunctionName,
list_of_states_to_save=list_of_states_to_convert,
ko_db_path_relative_to_db_connection_flex1=
'database/ko_db/unittest_ko_db/unittest_ko.db',
test_mode=True)
### DEFINE LOCAL CLASS VARIABLES NEEDED THROUGHOUT TESTING
cls.base_dir = 'base_connection_test_directory'
cls.create_file_path = 'base_connection_test_directory/test_createLocalFile'
cls.sub_script_dir = 'base_connection_test_directory/submission_script_test'
cls.move_fname = 'base_connection_test_rsync_remote_transfer_file.txt'
cls.move_dir1 = 'base_connection_test_directory/test_createLocalFile/test_directory1'
cls.move_dir2 = 'base_connection_test_directory/test_createLocalFile/test_directory2'
cls.test_localShellCommand_path = 'base_connection_test_directory/test_localShellCommand'
cls.test_localShellCommand_file = 'localShellCommand.file'
cls.tmp_local_data_storage_path = 'base_connection_test_directory/local_data_storage'
### CREATE DIRS FILES ETC FIR TESTS
# check that the base connection test directoy doesn't already exist
if os.path.isdir(cls.base_dir):
raise ValueError(
'The directory base_connection_test_directory must not exist, please move some where else.'
)
# create directories needed for test
pathlib.Path(cls.sub_script_dir).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir1).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.move_dir2).mkdir(parents=True, exist_ok=True)
pathlib.Path(cls.test_localShellCommand_path).mkdir(parents=True,
exist_ok=True)
pathlib.Path(cls.tmp_local_data_storage_path).mkdir(parents=True,
exist_ok=True)
# create a file for transfer tests
list_of_lines_of_file = ['This', 'is', 'a', 'test']
list_of_files_to_make = [
cls.move_dir1 + "/" + cls.move_fname,
cls.test_localShellCommand_path + '/' +
cls.test_localShellCommand_file
]
for file_name in list_of_files_to_make:
with open(file_name, mode='wt', encoding='utf-8') as myfile:
for line in list_of_lines_of_file:
myfile.write(line + "\n")
### CREATE REMOTE DIRECTORY THAT GETS DELETED EVERYTIME
mkdir_cmd = [
'mkdir -p ' + cls.base_path_on_cluster + '/' +
cls.relative_base_that_gets_deleted
]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand, mkdir_cmd)
if raw_out['return_code'] != 0:
print(
"Warning, could not create the remote dir. This currently shouldn't cause an error but may do later on. raw_out = ",
raw_out)
# make remote dirs
cls.tmp_mat_files = 'example_simulation'
cls.full_path_to_sim_example = cls.base_path_on_cluster + '/' + cls.relative_base_that_gets_deleted + '/' + cls.tmp_mat_files
mkdir_cmd = ['mkdir -p ' + cls.full_path_to_sim_example]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand, mkdir_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
"Whilst constructing the class we couldn't create remote directories!. raw_out = ",
raw_out)
# also add a whole simulation there
source_path = '/projects/flex1/database/mat_files/version73_example_mats_files/1/' # put the trailing slash so that it only copies the contents of the dir
transfer_cmd = [
'rsync -aP ' + source_path + ' ' + cls.full_path_to_sim_example
]
raw_out = cls.bg_conn.checkSuccess(cls.bg_conn.sendCommand,
transfer_cmd)
if raw_out['return_code'] != 0:
raise ValueError(
"Could not transfer example simulation data to the temporary directory! raw_out = ",
raw_out)
print('Setup up finishing!')