def get_processing_data_from_module(self, id_local_process):
"""
get processing (parallelization) data from module
function is run on remote computer
module name contain process id from local computer for its identification
restriction:
modules are not reloaded since only one operation on remote computer
:return:
"""
try:
processing = __import__("processing_" + id_local_process)
self.__processing.number_cpus = processing.number_cpus
self.__processing.mode = processing.mode
directory_temp = adapt_path("testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory))
remove_file(directory_temp + "processing_{}.py".format(id_local_process), False)
except Exception as err:
message(mode="ERROR", text="OS error: {}".format(err))
# remove byte code if it exists
try:
remove_file(directory_temp + "processing_{}.pyc".format(id_local_process), False)
except:
pass
def get_num_data_from_modules(self, id_local_process):
"""
get numerics data global, flow, mass, heat
function is run on remote computer
module names contain process id from local computer for its identification
delete modules after data obtained
restriction:
modules are not reloaded since only one operation on remote computer
:return:
"""
try:
directory_temp = adapt_path("{}testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory))
# first global to get process flags
self.get_num_data_for_process("global", id_local_process, directory_temp)
# processes
self.get_num_data_for_process("flow", id_local_process, directory_temp)
if self.__numerics_global.processes.mass_flag:
self.get_num_data_for_process("mass", id_local_process, directory_temp)
if self.__numerics_global.processes.heat_flag:
self.get_num_data_for_process("heat", id_local_process, directory_temp)
if self.__numerics_global.processes.deformation_flag:
self.get_num_data_for_process("deformation", id_local_process, directory_temp)
if self.__numerics_global.processes.fluid_momentum_flag:
self.get_num_data_for_process("fluid_momentum", id_local_process, directory_temp)
if self.__numerics_global.processes.overland_flag:
self.get_num_data_for_process("overland", id_local_process, directory_temp)
except Exception as err:
message(mode="ERROR", text="OS error: {0}".format(err))
def partition_mesh(self, directory):
"""
reloads prc.py to get parallelization data and writes *.pbs file
:param directory: (string)
:return:
"""
script_partition = "{}{}partition.sh".format(rootDirectory, adapt_path("testingEnvironment\\scripts\\"))
chdir(directory)
mesh_file = "{}{}.msh".format(directory, examplesName)
if not path.isfile(mesh_file): # no file *.msh, than look if one remained from previous mesh partitioning
mesh_file = "{}{}_mesh.txt".format(directory, examplesName)
if path.isfile(mesh_file):
try:
if self.__processing.mode == "mpi_elements": # for OGS_FEM_MPI, ...
Popen(
script_partition + " {} -e -asci {}".format(self.__processing.number_cpus, directory),
shell=True,
)
if self.__processing.mode == "mpi_nodes": # for OGS_FEM_PETSC
Popen(
"{} {} -n -binary {}".format(script_partition, self.__processing.number_cpus, directory),
shell=True,
)
except Exception as err:
message(mode="ERROR", text="{}".format(err))
else:
message(mode="ERROR", text="Mesh file missing")
def write_winscp_file(file_winscp, winscp_command_list, user, hostname, password, output_flag=True):
"""
:param file_winscp: (string)
:param winscp_command_list: (string list)
:param user: (string)
:param hostname: (string)
:param password: (string)
:param output_flag: (bool)
:return:
"""
file_winscp = adapt_path(file_winscp)
if output_flag:
message(mode='INFO', text=' Write winscp file')
try:
f = open(file_winscp, 'w')
except OSError as err:
message(mode='ERROR', text='OS error: {0}'.format(err))
else:
f.write('option batch abort \n')
f.write('option confirm off \n')
f.write('open sftp://{}:{}@{}/ \n'.format(user, password, hostname))
for command in winscp_command_list:
f.write('{} \n'.format(command))
f.write('exit')
f.close()
def upload_numerics_and_processing(subject, simulation_data):
"""
upload files with data for numerics and parallelization by calling winscp
then remove files from temp directory on local computer
:param item: (class Item)
:param simulation_data: (class Simulation Data)
:return:
"""
directory_local_temp = '{}{}'.format(rootDirectory, adapt_path('testingEnvironment\\scripts\\icbc\\temp\\'))
directory_remote_temp = '{}{}'.format(subject.directory_root, adapt_path_computer_selected(
'testingEnvironment\\scripts\\icbc\\temp\\', subject.operating_system))
module_list = list()
if simulation_data.read_file_flags.numerics:
module_list.clear()
module_list.append('numerics_global')
module_list.append('numerics_flow')
if simulation_data.numerics_global.processes.mass_flag:
module_list.append('numerics_mass')
if simulation_data.numerics_global.processes.heat_flag:
module_list.append('numerics_heat')
if simulation_data.numerics_global.processes.deformation_flag:
module_list.append('numerics_deformation')
if simulation_data.numerics_global.processes.fluid_momentum_flag:
module_list.append('numerics_fluid_momentum')
if simulation_data.numerics_global.processes.overland_flag:
module_list.append('numerics_overland')
upload_modules(subject, module_list, directory_local_temp, directory_remote_temp)
if simulation_data.read_file_flags.processing:
module_list.clear()
module_list.append('processing')
upload_modules(subject, module_list, directory_local_temp, directory_remote_temp)
def __init__(self, subject, item_type, item_case, item_configuration, flow_process, element_type):
self.__directory_repository = adapt_path( # independent of configuration (one folder for all)
'{}testingEnvironment\\scripts\\icbc\\repository\\{}\\{}\\{}\\{}\\'.format(
rootDirectory, item_type, item_case, flow_process, element_type))
Test.__init__(self, subject, item_type, item_case, item_configuration, flow_process, element_type, adapt_path(
'{}examples\\files\\{}\\{}\\{}\\{}\\{}\\'.format(
subject.directory, item_type, item_case, flow_process, element_type, item_configuration)))
def compare_results_with_reference(self):
"""
compare results with results in reference folder for regression tests
if file disagrees, add file name to self._subject.directory + 'references\\deviatingFiles.log'
then call record_differences_between_files(), which writes deviations themselves into
directory_reference + 'deviations.log'
configurationCustomized.outputFile are not compared
:return:
"""
message(mode='INFO', text='Compare result with reference files\n {}'.format(self._item.name()))
directory_reference = adapt_path(
"{}references\\{}\\{}\\{}\\{}\\{}\\".format(
self._subject.directory, self._item.type, self._item.case,
self._item.flow_process, self._item.element_type, self._item.configuration))
if path.exists(directory_reference):
if path.exists(self._item.directory):
# clear file that will contain regressions (differences between file contents)
remove_file('{}deviations.log'.format(directory_reference), False)
for file_name in listdir(directory_reference):
if file_name != outputFile and file_name != 'deviations.log':
# file to check
result_comparison_flag = compare_files(self._item.directory + file_name,
directory_reference + file_name)
if not result_comparison_flag:
# file contents disagree
message(mode='INFO', text='Deviating file: {}'.format(file_name))
append_to_file(
adapt_path('{}references\\deviatingFiles_{}.log'.format(
self._subject.directory, self._item.configuration)),
'{}{}\n'.format(self._item.directory, file_name))
record_regression_of_file(file_name, self._item.directory, directory_reference,
'deviations.log', output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
else:
message(mode='ERROR', text='Directory with reference files missing')
def select(self, setting_inst):
"""
:param setting_inst:
:return:
"""
if not self.__computer:
self.__computer = setting_inst.get_name_list('computer')[0]
# only one entry in list here and in the following
if not self.__superuser:
if not self.__user:
self.__user = setting_inst.get_name_list('users')[0]
else:
self.__user = setting_inst.query_username(self.__superuser, self.__computer)
if not self.__code:
self.__code = setting_inst.get_name_list('codes')[0]
if not self.__branch:
self.__branch = setting_inst.get_name_list('branches')[0]
if location == 'local':
self.__rootDirectory = setting_inst.query_directory_root(self.__computer, self.__user)
self.__location = setting_inst.query_location(self.__computer)
self.__operating_system = setting_inst.query_operating_system(self.__computer)
self.__directory = adapt_path('{}testingEnvironment\\{}\\{}\\{}\\'.format(
self.__rootDirectory, self.__computer, self.__code, self.__branch))
#self.__gateDirectory = adapt_path(self.__rootDirectory + 'testingEnvironment\\'
# + self.__computer + '\\gate\\')
self.__plotDirectory = adapt_path('{}testingEnvironment\\{}\\{}\\{}\\examples\\plots\\'.format(
rootDirectory, self.__computer, self.__code, self.__branch))
self.__hostname = setting_inst.query_hostname(self.__computer)
else:
self.__location = 'remote'
self.__directory = adapt_path('{}testingEnvironment\\{}\\{}\\{}\\'.format(
rootDirectory, self.__computer, self.__code, self.__branch))
self.__gateDirectory = adapt_path('{}testingEnvironment\\{}\\gate\\'.format(rootDirectory, self.__computer))
def __init__(self, subject, item_type, item_case, item_configuration, flow_process, element_type):
example = '{}\\'.format(item_type)
if item_case:
example = '{}{}\\{}\\{}\\'.format(example, item_case, flow_process, element_type)
if item_configuration:
example = '{}{}\\'.format(example, item_configuration)
directory_local = adapt_path('{}testingEnvironment\\{}\\{}\\{}\\examples\\files\\{}'.format(
rootDirectory, subject.computer, subject.code, subject.branch, example))
self.__directory_computer_selected = adapt_path_computer_selected('{}examples\\files\\{}'.format(
subject.directory, example), subject.operating_system)
Test.__init__(self, subject, item_type, item_case, item_configuration,
flow_process, element_type, directory_local)
def operate(subject, item, operation_type, operation, simulation_data):
"""
:param subject: (class Subject)
:param item: (class Item)
:param operation_type: (one-char string)
:param operation: (one-char string)
:param simulation_data: (string)
:return:
"""
# the naming allows to upload several shell_scripts by separate icbc instances
if operation_type == 's': # simulating
shell_script = adapt_path(rootDirectory + 'testingEnvironment\\scripts\\icbc\\temp\\remoteRun' + '_' +
operation_type + '_' + operation + '_' + item.type + '_' +
item.case + '_' + item.configuration + '.sh')
else: # building depends only on configuration (plotting is always local)
shell_script = adapt_path(rootDirectory + 'testingEnvironment\\scripts\\icbc\\temp\\remoteRun' + '_' +
operation_type + '_' + operation + '_' + item.configuration + '.sh')
upload_files(item, simulation_data)
write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script)
execute_shell_script_on_remote_computer(subject, shell_script)
def operate(subject, item, operation_type, operation, simulation_data):
"""
:param subject: (class Subject)
:param item: (class Item)
:param operation_type: (one-char string)
:param operation: (one-char string)
:param simulation_data: (string)
:return:
"""
shell_script = '{}{}remoteRun_{}.sh'.format(
rootDirectory, adapt_path('testingEnvironment\\scripts\\icbc\\temp\\'), getpid())
upload_numerics_and_processing(subject, simulation_data)
write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script)
execute_shell_script_on_remote_computer(subject, shell_script)
def configure(self, operation_inst):
"""
0. call function to set flags for file uploads
1. if operation is to compare results with references, clear log file with names of
deviating files from previous runs
2. check if list entries for test items exit
:param operation_inst: (class Building, Simulating, Plotting (Operation))
:return: 0 if success; 1 if lists for item not complete
"""
operation_inst.set_upload_file_flags()
if operation_inst.selected_operation_type == 's' and operation_inst.selected_operation == 'o':
# the selected operation is to compare results with references
files = glob(adapt_path('{}references\\deviatingFiles*'.format(self.__subject_inst.directory)))
for file in files:
remove_file(file, False)
def get_built_file_for_release(self, item):
"""
give built file a name, e.g. ogs_kb1_Linux_OGS_FEM and return it with path
:param item: (class Item)
:return: (string) path and name of built file for release, '1' if OP not supported
"""
if system() == 'Windows':
extension = '.exe'
elif system() == 'Linux':
extension = ''
else:
message(mode='ERROR', not_supported=system())
return '1'
return adapt_path('{}releases\\{}_{}_{}_{}{}'.format(
self.__directory, self.__code, self.__branch, system(), item.configuration, extension))
def call_winscp(winscp_command_list, user, hostname, password, output_flag=True):
"""
file is marked and identified with process id
:param winscp_command_list:
:param user:
:param hostname:
:param password:
:param output_flag:
:return:
"""
temp_winscp_file = '{}{}winscp_file_{}.txt'.format(rootDirectory, adapt_path(
'testingEnvironment\\scripts\\icbc\\temp\\'), getpid())
write_winscp_file(temp_winscp_file, winscp_command_list, user, hostname, password, output_flag)
try:
check_call('{} /script={}'.format(winscp, temp_winscp_file))
except Exception as err:
message(mode='ERROR', text='{0}'.format(err))
remove_file(temp_winscp_file, False)
print('\n')
def show_regression(self):
"""
reads file where regression is documented
:return:
"""
message(mode='INFO', text='Show regression {} {}'.format(system(), self._item.configuration))
try:
file = adapt_path('{}\\testingEnvironment\\{}\\{}\\{}\\references\\deviatingFiles_{}.log'.format(
rootDirectory, self._subject.computer,
self._subject.code, self._subject.branch, self._item.configuration))
try:
log = open(file, 'r')
message(mode='INFO', text=' Regression in files')
for line in log:
print(line)
except:
message(mode='INFO', text=' No regression recorded')
# directory = adapt_path('{}\\testingEnvironment\\scripts\\icbc\\shared\\'.format(rootDirectory))
# call('py.test {}test_regression.py'.format(directory), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
def show_regression(self):
"""
reads file where regression is documented
:return:
"""
message(mode='INFO', text='Show regression\n {}'.format(self._item.name()))
try:
file = adapt_path("{}references\\{}\\{}\\{}\\{}\\{}\\deviations.log".format(
self._subject.directory, self._item.type, self._item.case,
self._item.flow_process, self._item.element_type, self._item.configuration))
try:
log = open(file, 'r')
message(mode='INFO', text=' Regression recorded:')
for line in log:
print(line)
except:
message(mode='INFO', text=' No regression recorded')
# directory = adapt_path('{}\\testingEnvironment\\scripts\\icbc\\shared\\'.format(rootDirectory))
# call('py.test {}test_regression.py'.format(directory), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
def store_results_as_reference(self):
"""
copy results (these are files with endings configurationShared.outputFileEndings)
into reference folder
(generates folder for references if it does not exist)
each computer, case, branch has own reference folder
:return:
"""
message(mode='INFO', text='Store results as reference \n {}'.format(self._item.name()))
directory_reference = adapt_path("{}references\\{}\\{}\\{}\\{}\\{}\\".format(
self._subject.directory, self._item.type, self._item.case,
self._item.flow_process, self._item.element_type, self._item.configuration))
generate_folder(directory_reference)
if path.exists(self._item.directory):
for extension in outputFileEndings:
for file_name in listdir(self._item.directory):
if file_name.endswith('.{}'.format(extension)):
copy_file(self._item.directory + file_name, directory_reference)
else:
message(mode='ERROR', text='Directory missing')
def __init__(self, subject, configuration):
Item.__init__(self, subject, configuration, adapt_path('{}Build_{}\\'.format(subject.directory, configuration)))
