def get_processing_data_from_module(self, id_local_process):
"""
get processing (parallelization) data from module
function is run on remote computer
module name contain process id from local computer for its identification
restriction:
modules are not reloaded since only one operation on remote computer
:return:
"""
try:
processing = __import__("processing_" + id_local_process)
self.__processing.number_cpus = processing.number_cpus
self.__processing.mode = processing.mode
directory_temp = adapt_path("testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory))
remove_file(directory_temp + "processing_{}.py".format(id_local_process), False)
except Exception as err:
message(mode="ERROR", text="OS error: {}".format(err))
# remove byte code if it exists
try:
remove_file(directory_temp + "processing_{}.pyc".format(id_local_process), False)
except:
pass
def clear_folder(self):
"""
delete *.tec, *.txt, *.asc, *plt and results.tar (local folder for remote computer)
:return:
"""
message(mode='INFO', text='Clear plotting folder ' + self._item.name())
#
remove_file("{}results_{}.jpg".format(self._subject.directory_plot, self._item.type), False)
file_endings = list() # concerns files with outputFileEndings and *.plt
for ending in outputFileEndings:
file_endings.append(ending)
file_endings.append('plt')
if path.exists(self._item.directory):
for file in listdir(self._item.directory):
for ending in file_endings:
if file.endswith('.{}'.format(ending)):
if self._subject.location == 'remote' or ending != 'tec':
# do not delete tec if local
remove_file(self._item.directory + file, False)
# myLocalTarFile = self._item.directory + 'results.tar' # done in get_results()
# if path.isfile(myLocalTarFile):
# remove(myLocalTarFile)
else:
message(mode='ERROR', text='Directory missing')
def get_results(self):
"""
download and unpack tar file from remote to local computer,
if local computer is windows, convert files into dos-format
:return:
"""
if self._subject.location == 'remote':
message(mode='INFO', text='Get results {}'.format(self._item.name()))
mod = __import__(self._subject.computer)
generate_folder(self._item.directory)
if path.exists(self._item.directory):
# clear local directory
files = glob(self._item.directory + '*')
for file in files:
remove_file(file)
winscp_command = 'get {}results.tar {}'.format(
self._item.directory_computer_selected, self._item.directory)
call_winscp([winscp_command], self._subject.user, self._subject.hostname, mod.pwd, output_flag=True)
unpack_tar_file(self._item.directory) # tar file on local computer
if system() == 'Windows':
message(mode='INFO', text=' Convert to dos')
for file in listdir(self._item.directory):
unix2dos(file, output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
else:
message(mode='INFO', text='{} is local - Nothing done'.format(self._subject.computer))
def generate_jpgs(self):
"""
generate JPG with tecplot
1. generate tecplot macro
2. call tecplot via subprocess
3. remove tecplot macro
:return:
"""
message(mode='INFO', text='Generate Jpgs ' + self._item.type)
if path.exists(self._subject.directory_plot):
chdir(self._subject.directory_plot)
for file in glob("{}*".format(self._item.type)):
filename = path.splitext(file)[0]
message(mode='INFO', text=' {}'.format(filename))
write_tecplot_macro_for_jpg(self._subject.directory_plot, filename)
layout = '{}{}.lay'.format(self._subject.directory_plot, filename)
try:
call('{} {} -b -p {}_genJPG.mcr'.format(tecplot, layout, self._subject.directory_plot), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
remove_file('{}_genJPG.mcr'.format(self._subject.directory_plot), output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
def clear_folder(self):
"""
delete built files (from folder where they are after compilation)
:return:
"""
message(mode='INFO', text='Removing release {} {}'.format(system(), self._item.configuration))
remove_file(self._subject.get_built_file(self._item), False)
def configure(self, operation_inst):
"""
0. call function to set flags for file uploads
1. if operation is to compare results with references, clear log file with names of
deviating files from previous runs
2. check if list entries for test items exit
:param operation_inst: (class Building, Simulating, Plotting (Operation))
:return: 0 if success; 1 if lists for item not complete
"""
operation_inst.set_upload_file_flags()
if operation_inst.selected_operation_type == 's' and operation_inst.selected_operation == 'o':
# the selected operation is to compare results with references
files = glob(adapt_path('{}references\\deviatingFiles*'.format(self.__subject_inst.directory)))
for file in files:
remove_file(file, False)
def call_winscp(winscp_command_list, user, hostname, password, output_flag=True):
"""
file is marked and identified with process id
:param winscp_command_list:
:param user:
:param hostname:
:param password:
:param output_flag:
:return:
"""
temp_winscp_file = '{}{}winscp_file_{}.txt'.format(rootDirectory, adapt_path(
'testingEnvironment\\scripts\\icbc\\temp\\'), getpid())
write_winscp_file(temp_winscp_file, winscp_command_list, user, hostname, password, output_flag)
try:
check_call('{} /script={}'.format(winscp, temp_winscp_file))
except Exception as err:
message(mode='ERROR', text='{0}'.format(err))
remove_file(temp_winscp_file, False)
print('\n')
def compare_results_with_reference(self):
"""
compare results with results in reference folder for regression tests
if file disagrees, add file name to self._subject.directory + 'references\\deviatingFiles.log'
then call record_differences_between_files(), which writes deviations themselves into
directory_reference + 'deviations.log'
configurationCustomized.outputFile are not compared
:return:
"""
message(mode='INFO', text='Compare result with reference files\n {}'.format(self._item.name()))
directory_reference = adapt_path(
"{}references\\{}\\{}\\{}\\{}\\{}\\".format(
self._subject.directory, self._item.type, self._item.case,
self._item.flow_process, self._item.element_type, self._item.configuration))
if path.exists(directory_reference):
if path.exists(self._item.directory):
# clear file that will contain regressions (differences between file contents)
remove_file('{}deviations.log'.format(directory_reference), False)
for file_name in listdir(directory_reference):
if file_name != outputFile and file_name != 'deviations.log':
# file to check
result_comparison_flag = compare_files(self._item.directory + file_name,
directory_reference + file_name)
if not result_comparison_flag:
# file contents disagree
message(mode='INFO', text='Deviating file: {}'.format(file_name))
append_to_file(
adapt_path('{}references\\deviatingFiles_{}.log'.format(
self._subject.directory, self._item.configuration)),
'{}{}\n'.format(self._item.directory, file_name))
record_regression_of_file(file_name, self._item.directory, directory_reference,
'deviations.log', output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
else:
message(mode='ERROR', text='Directory with reference files missing')
def clear_folder(self):
"""
delete files
1. configurationShared.outputFileEndings (*.tec, *.txt, *.plt, *.vtk)
2. results.tar
3. configurationShared.outputFile
(remote folder for remote computer)
:return:
"""
message(mode='INFO', text='Clear simulation folder \n {}'.format(self._item.name()))
#
file_endings = list() # concerns files with outputFileEndings and *.plt
for ending in outputFileEndings:
file_endings.append(ending)
file_endings.append('plt')
if path.exists(self._item.directory):
for file in listdir(self._item.directory):
for ending in file_endings:
if file.endswith('.{}'.format(ending)):
remove_file(self._item.directory + file, False)
tarfile_remote = '{}results.tar'.format(self._item.directory)
remove_file(tarfile_remote, False)
output_file = self._item.directory + outputFile
remove_file(output_file, False)
else:
message(mode='ERROR', text='Directory missing')
def upload_modules(subject, module_list, directory_local, directory_remote):
"""
upload modules list and remove them from local directory
:param subject: (string)
:param module_list: (string list)
:param directory_local: (string)
:param directory_remote: (string)
:return:
"""
mod = __import__(subject.computer) # module with password for remote computer
command_list = list()
for module in module_list:
command_list.append('put {0}{2}_{3}.py {1}{2}_{3}.py'.format(
directory_local, directory_remote, module, getpid()))
try:
call_winscp(command_list, subject.user, subject.hostname, mod.pwd, False)
for module in module_list:
remove_file('{}{}'.format(directory_local, module), False)
except Exception as err:
message(mode='ERROR', text='{0}'.format(err))
def get_num_data_for_process(self, process_name, id_local_process, directory_temp):
module = __import__("numerics_{}_{}".format(process_name, id_local_process))
if process_name == "global":
self.__numerics_global.coupled_flag = str2bool(module.coupled_flag)
self.__numerics_global.lumping_flag = str2bool(module.lumping_flag)
self.__numerics_global.non_linear_flag = str2bool(module.non_linear_flag)
self.__numerics_global.processes.flow = module.flow_process
self.__numerics_global.processes.mass_flag = str2bool(module.mass_flag)
self.__numerics_global.processes.heat_flag = str2bool(module.heat_flag)
self.__numerics_global.processes.deformation_flag = str2bool(module.deformation_flag)
self.__numerics_global.processes.fluid_momentum_flag = str2bool(module.fluid_momentum_flag)
self.__numerics_global.processes.overland_flag = str2bool(module.overland_flag)
print("def: " + str(self.__numerics_global.processes.deformation_flag))
else:
self.__solver_dir[process_name] = module.solver
self.__preconditioner_dir[process_name] = module.precond
self.__theta_dir[process_name] = module.theta
remove_file("{}numerics_{}_{}.py".format(directory_temp, process_name, id_local_process), False)
try: # remove byte code if it exists
remove_file(directory_temp + "numerics_{}_{}.pyc".format(process_name, id_local_process), False)
except:
pass
def cleanup():
util.remove_file(file_string)