def item_is_to_test(self, item_case):
"""
check test_mode, state,
used for simulation, plotting operation type (case item exists)
(all items (i.e. configurations) involved in building operation type)
:param item_case: (string or None)
:return: (bool )False: not involved, True: involved or ERROR
"""
execute_flag = True
if location == 'local' and item_case:
if self.__setting_inst.test_mode == '0':
# no test mode - select everything by typing on console
# and than the items in the loop (in environment run) is involved
pass
elif self.__setting_inst.test_mode == '1':
# via browser
execute_flag = self.__setting_inst.query_column_entry_for_name('cases', item_case, 'state')
elif self.__setting_inst.test_mode == '2':
# via CI tool
if not self.__setting_inst.query_column_entry_for_name('cases', item_case, 'active'):
execute_flag = False
else:
message(mode='ERROR', not_supported=self.__setting_inst.test_mode) # returns '1'
return execute_flag # remote always '1'
def query_id_for_name(self, table, name):
"""
query an id from a table for a given name
requirement:
table columns must be: id, name, ...
name must be unique in table (not checked)
:param table: (string): name of table in SQL schema
:param name: (string): entry in column name
:return: id (string) or '-1' if no id found (exception)
"""
if not name: # empty string
return None # no name given get_table_data_from_database
else:
# query_text = "SELECT {} FROM computer WHERE name=%s"
# .format(table, name)
# .encode(encoding='UTF-8', errors='strict')
# query_text2 = query_text.decode(encoding='UTF-8', errors='strict')
# .format(table, name)
# row_list = self.query("SELECT id FROM " + str(table) + " WHERE name=%s", [name])
query_text = "SELECT id from {} WHERE name='{}'".format(table, name)
row_list = self.query(query_text)
if row_list[0] is not None:
return str(row_list[0]['id']) # each row_list entry is dictionary {'id': (string)}
else:
message(mode='ERROR', text='Column id for {} not found in table {}'.format(name, table))
return '-1' # exception
def clear_folder(self):
"""
delete files
1. configurationShared.outputFileEndings (*.tec, *.txt, *.plt, *.vtk)
2. results.tar
3. configurationShared.outputFile
(remote folder for remote computer)
:return:
"""
message(mode='INFO', text='Clear simulation folder \n {}'.format(self._item.name()))
#
file_endings = list() # concerns files with outputFileEndings and *.plt
for ending in outputFileEndings:
file_endings.append(ending)
file_endings.append('plt')
if path.exists(self._item.directory):
for file in listdir(self._item.directory):
for ending in file_endings:
if file.endswith('.{}'.format(ending)):
remove_file(self._item.directory + file, False)
tarfile_remote = '{}results.tar'.format(self._item.directory)
remove_file(tarfile_remote, False)
output_file = self._item.directory + outputFile
remove_file(output_file, False)
else:
message(mode='ERROR', text='Directory missing')
def write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script):
"""
:param subject: (Subject)
:param item: (Item)
:param operation_type: (one-char string)
:param operation: (one-char string)
:param shell_script: (string)
:return:
"""
try:
f = open(shell_script, 'w')
except Exception as err:
message(mode='ERROR', text='{0}'.format(err))
else:
f.write('#!/bin/sh\n')
f.write('module load Python-3.3.2\n')
f.write('python3 {}testingEnvironment/scripts/icbc/customized/run_remote.py '.format(subject.directory_root))
f.write('{} {} {} {} {} '.format(subject.computer, subject.user, subject.code, subject.branch, getpid()))
if operation_type == 'b': # building
f.write('None None {} None None '.format(item.configuration))
elif operation_type == 's': # simulating
f.write('{} {} {} {} {} '.format(
item.type, item.case, item.configuration, item.flow_process, item.element_type))
elif operation_type == 'p': # plotting
f.write(' {} {} {} None None '.format(item.type, item.case, item.configuration))
f.write('{} {}'.format(operation_type, operation))
f.close()
def clear_folder(self):
"""
delete *.tec, *.txt, *.asc, *plt and results.tar (local folder for remote computer)
:return:
"""
message(mode='INFO', text='Clear plotting folder ' + self._item.name())
#
remove_file("{}results_{}.jpg".format(self._subject.directory_plot, self._item.type), False)
file_endings = list() # concerns files with outputFileEndings and *.plt
for ending in outputFileEndings:
file_endings.append(ending)
file_endings.append('plt')
if path.exists(self._item.directory):
for file in listdir(self._item.directory):
for ending in file_endings:
if file.endswith('.{}'.format(ending)):
if self._subject.location == 'remote' or ending != 'tec':
# do not delete tec if local
remove_file(self._item.directory + file, False)
# myLocalTarFile = self._item.directory + 'results.tar' # done in get_results()
# if path.isfile(myLocalTarFile):
# remove(myLocalTarFile)
else:
message(mode='ERROR', text='Directory missing')
def clear_folder(self):
"""
delete built files (from folder where they are after compilation)
:return:
"""
message(mode='INFO', text='Removing release {} {}'.format(system(), self._item.configuration))
remove_file(self._subject.get_built_file(self._item), False)
def wait(self):
"""
wait until release exists
:return:
"""
message(mode='INFO', text='Waiting for release {} {}'.format(system(), self._item.configuration))
wait_for_file(self._subject.get_built_file(self._item))
def write_winscp_file(file_winscp, winscp_command_list, user, hostname, password, output_flag=True):
"""
:param file_winscp: (string)
:param winscp_command_list: (string list)
:param user: (string)
:param hostname: (string)
:param password: (string)
:param output_flag: (bool)
:return:
"""
file_winscp = adapt_path(file_winscp)
if output_flag:
message(mode='INFO', text=' Write winscp file')
try:
f = open(file_winscp, 'w')
except OSError as err:
message(mode='ERROR', text='OS error: {0}'.format(err))
else:
f.write('option batch abort \n')
f.write('option confirm off \n')
f.write('open sftp://{}:{}@{}/ \n'.format(user, password, hostname))
for command in winscp_command_list:
f.write('{} \n'.format(command))
f.write('exit')
f.close()
def delete_store(_id):
store = StoreModel.find_store_by_id(_id)
if store:
name = store.name
store.delete_from_db()
return message('The {} was deleted from database'.format(name), 200)
return message("There is no store whose id is equal {}".format(_id), 400)
def select_items_to_test(self, operation_type, operation, computer):
"""
generate examples Lists to loop over in Environment run
:param operation_type:
:param operation:
:param computer:
:return:
"""
if operation_type == 'b': # building (only configuration)
self.__item_constituents.type_list = [None]
self.__item_constituents.case_list = [[None]]
self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test(
level='configurations', computer_of_subject=computer)
elif operation_type == 's': # simulating (all item constituents (type, case, configuration))
self.__item_constituents.type_list = self.select_constituent_of_items_to_test(level='types')
self.__item_constituents.case_list = self.select_constituent_of_items_to_test(level='cases')
self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test(
level='configurations', computer_of_subject=computer)
elif operation_type == 'p': # plotting
self.__item_constituents.type_list = self.select_constituent_of_items_to_test(level='types')
if operation == 'j' or operation == 'w': # generate or wait for JPG (only type)
self.__item_constituents.case_list = [[None]]
self.__item_constituents.configuration_list = [None]
else: # all item constituents (type, case, configuration)
self.__item_constituents.case_list = self.select_constituent_of_items_to_test(level='cases')
self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test(
level='configurations', computer_of_subject=computer)
else:
message(mode='ERROR', not_supported='operation_type {}'.format(operation_type))
def select_constituent_of_items_to_test(self, level, computer_of_subject=None):
"""
calls member function selectNames to get item constituents
if they are not previously selected (i.e. in environment constructor or anywhere in last operations)
:param level:
:param computer_of_subject:
:return: list (string) for one item constituent (meaning: types, cases, configuration)
nested list if table = 'cases'
"""
if level == 'types':
if self.__item_constituents.type_list == [None]:
# len(self.__item_constituents.type_list) == 0 or not self.__item_constituents.type_list[0]:
return self.get_name_list(table='types')
else:
return self.__item_constituents.type_list
elif level == 'cases':
if self.__item_constituents.case_list == [[None]]:
# len(self.__item_constituents.case_list) == 0 or self.__item_constituents.case_list[0] == [None]:
return self.get_name_list(table='cases')
else:
return self.__item_constituents.case_list
elif level == 'configurations':
if self.__item_constituents.configuration_list == [None]:
# len(self.__item_constituents.configuration_list) == 0 \
# or not self.__item_constituents.configuration_list[0]:
return self.get_name_list(table='configurations', computer=computer_of_subject)
else:
return self.__item_constituents.configuration_list
else:
message(mode='ERROR', not_supported='level ' + level)
def id2name(self, table, id_selected, lists_inst):
"""
Convert id to name (list) and return this as a nested list [['...']]
class member to have access to __selectedTypeIdList
:param table:
:param id_selected:
:param lists_inst:
:return:
"""
if id_selected == 'a':
if table == 'cases':
lists_inst.name.append(deepcopy(lists_inst.nameSub))
return lists_inst.name
else:
return lists_inst.nameSub
elif id_selected == 'r':
if table == 'types':
for id_running in self.__selectedTypeIdList:
lists_inst.name.append(deepcopy(lists_inst.nameSub[get_list_id(id_running, lists_inst.id)]))
return lists_inst.name
else:
message(mode='ERROR', text='Range supported only for types')
return None
else: # single item
lists_inst.name.append(deepcopy(lists_inst.nameSub[get_list_id(id_selected, lists_inst.id)]))
if table == 'cases':
name_list2 = list()
name_list2.append(deepcopy(lists_inst.name))
return name_list2
else:
return lists_inst.name
def write_num(self):
"""
call member of SimulationData to write file *.num
:return:
"""
message(mode='INFO', text='Write *.num {}'.format(self._item.name()))
self._simulation_data.write_num(self._item.directory)
def partition_mesh(self, directory):
"""
reloads prc.py to get parallelization data and writes *.pbs file
:param directory: (string)
:return:
"""
script_partition = "{}{}partition.sh".format(rootDirectory, adapt_path("testingEnvironment\\scripts\\"))
chdir(directory)
mesh_file = "{}{}.msh".format(directory, examplesName)
if not path.isfile(mesh_file): # no file *.msh, than look if one remained from previous mesh partitioning
mesh_file = "{}{}_mesh.txt".format(directory, examplesName)
if path.isfile(mesh_file):
try:
if self.__processing.mode == "mpi_elements": # for OGS_FEM_MPI, ...
Popen(
script_partition + " {} -e -asci {}".format(self.__processing.number_cpus, directory),
shell=True,
)
if self.__processing.mode == "mpi_nodes": # for OGS_FEM_PETSC
Popen(
"{} {} -n -binary {}".format(script_partition, self.__processing.number_cpus, directory),
shell=True,
)
except Exception as err:
message(mode="ERROR", text="{}".format(err))
else:
message(mode="ERROR", text="Mesh file missing")
def wait(self):
"""
wait until jpg file is in folder
:return:
"""
message(mode='INFO', text='Waiting for jpg file {}'.format(self._item.type))
wait_for_file("{}results_{}.jpg".format(self._subject.directory_plot, self._item.type))
def write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script):
"""
:param subject: (Subject)
:param item: (Item)
:param operation_type: (one-char string)
:param operation: (one-char string)
:param shell_script: (string)
:return:
"""
try:
f = open(shell_script, 'w')
except Exception as err:
message(mode='ERROR', text='{0}'.format(err))
else:
f.write('#!/bin/sh\n')
f.write('module load python3.3\n')
f.write('python ' + subject.directory_root + 'testingEnvironment/scripts/icbc/customized/run_remote.py ')
f.write(subject.computer + ' ' + subject.user + ' ' + subject.code + ' ' + subject.branch + ' ')
if operation_type == 'b': # building
f.write('No No ' + item.configuration + ' ')
else:
f.write(item.type + ' ' + item.case + ' ' + item.configuration + ' ')
f.write(operation_type + ' ' + operation)
f.close()
def get_processing_data_from_module(self, id_local_process):
"""
get processing (parallelization) data from module
function is run on remote computer
module name contain process id from local computer for its identification
restriction:
modules are not reloaded since only one operation on remote computer
:return:
"""
try:
processing = __import__("processing_" + id_local_process)
self.__processing.number_cpus = processing.number_cpus
self.__processing.mode = processing.mode
directory_temp = adapt_path("testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory))
remove_file(directory_temp + "processing_{}.py".format(id_local_process), False)
except Exception as err:
message(mode="ERROR", text="OS error: {}".format(err))
# remove byte code if it exists
try:
remove_file(directory_temp + "processing_{}.pyc".format(id_local_process), False)
except:
pass
def get_num_data_from_modules(self, id_local_process):
"""
get numerics data global, flow, mass, heat
function is run on remote computer
module names contain process id from local computer for its identification
delete modules after data obtained
restriction:
modules are not reloaded since only one operation on remote computer
:return:
"""
try:
directory_temp = adapt_path("{}testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory))
# first global to get process flags
self.get_num_data_for_process("global", id_local_process, directory_temp)
# processes
self.get_num_data_for_process("flow", id_local_process, directory_temp)
if self.__numerics_global.processes.mass_flag:
self.get_num_data_for_process("mass", id_local_process, directory_temp)
if self.__numerics_global.processes.heat_flag:
self.get_num_data_for_process("heat", id_local_process, directory_temp)
if self.__numerics_global.processes.deformation_flag:
self.get_num_data_for_process("deformation", id_local_process, directory_temp)
if self.__numerics_global.processes.fluid_momentum_flag:
self.get_num_data_for_process("fluid_momentum", id_local_process, directory_temp)
if self.__numerics_global.processes.overland_flag:
self.get_num_data_for_process("overland", id_local_process, directory_temp)
except Exception as err:
message(mode="ERROR", text="OS error: {0}".format(err))
def delete_item(_id):
item = ItemModel.find_item_by_id(_id)
if item:
name = item.name
item.delete_from_db()
return message('The {} was deleted from database'.format(name), 200)
return message("There is no item whose id is equal {}".format(_id), 400)
def write_num(self, directory):
"""
Reads global numerics and parallelization data from files and writes *.num file for simulation
:param directory:
:return:
"""
try:
file_stream = open(directory + examplesName + ".num", "w")
except OSError as err:
message(mode="ERROR", text="OS error: {0}".format(err))
else:
if self.__numerics_global.coupled_flag:
file_stream.write("\n$OVERALL_COUPLING\n")
file_stream.write(" {} {}\n".format(coupling_iterations_min, coupling_iterations_max))
self.write_process_into_numerics_file(file_stream, "flow")
if self.__numerics_global.processes.mass_flag:
self.write_process_into_numerics_file(file_stream, "mass")
if self.__numerics_global.processes.heat_flag:
self.write_process_into_numerics_file(file_stream, "heat")
if self.__numerics_global.processes.deformation_flag:
self.write_process_into_numerics_file(file_stream, "deformation")
if self.__numerics_global.processes.fluid_momentum_flag:
self.write_process_into_numerics_file(file_stream, "fluid_momentum")
if self.__numerics_global.processes.overland_flag:
self.write_process_into_numerics_file(file_stream, "overland")
file_stream.write("\n#STOP\n")
file_stream.close()
def query_ids_from_column_entries(self, table, column_name, entry):
"""
...
:param table: (string): name of table in SQL schema
:param item_id: (string): entry in id column (primary key)
:param column_name: (string) name of column where entry is searched from
:param entry: (string)
:return: (string) column entry, '-1' if no entry found (exception)
"""
result_list = list()
row_list = self.query("SELECT t.id FROM {} t WHERE t.{}={}".format(table, column_name, entry))
try:
a = row_list[0]
except:
message(mode='ERROR', text='Column entry {} of {} not found'.format(entry, column_name))
return '-1' # exception
else:
i = 0
while True:
try:
a = row_list[i]
except:
break
else:
result_list.append(row_list[i]['id'])
i += 1
return result_list
def partition_mesh(self):
"""
call member of SimulationData to partition shell script
:return:
"""
message(mode='INFO', text='Mesh partition {}'.format(self._item.name()))
self._simulation_data.partition_mesh(self._item.directory)
def write_pbs(self):
"""
call member of SimulationData to write file run.pbs
:return:
"""
message(mode='INFO', text='Write run.pbs {}'.format(self._item.name()))
self._simulation_data.write_pbs(self._item.directory,
self._subject.get_built_file(self._item), self._item.type)
def wait(self):
"""
Wait until configurationShared.outputFile exists
:return:
"""
message(mode='INFO', text='Waiting for output file {}'.format(self._item.name()))
#
wait_for_file(self._item.directory + outputFile)
def write_pbs(self, directory, executable, item_type):
"""
reloads prc.py to get parallelization data and writes *.pbs file
:param directory:
:param executable:
:param item_type:
:return:
"""
# config
omp_threads = None
if self.__processing.mode == "sequential":
ncpus = "1"
command = " "
place = "group=host"
elif self.__processing.mode == "omp":
ncpus = self.__processing.number_cpus
omp_threads = self.__processing.number_cpus
command = " "
place = "group=host"
elif self.__processing.mode == "mpi_elements" or self.__processing.mode == "mpi_nodes": # parallel
ncpus = self.__processing.number_cpus
command = "mpirun -r rsh -machinefile $PBS_NODEFILE -n " + ncpus + " "
place = "scatter"
else:
message(mode="ERROR", not_supported="Mode {}".format(self.__processing.mode))
return
# write
try:
file_stream = open("{}run.pbs".format(directory), "w")
except OSError as err:
message(mode="ERROR", text="OS error: {}".format(err))
else:
file_stream.write("#!/bin/bash\n")
file_stream.write("#PBS -o {}screenout.txt\n".format(directory))
file_stream.write("#PBS -j oe\n")
file_stream.write("#PBS -r n\n")
file_stream.write("#PBS -l walltime={}\n".format(walltime))
file_stream.write("#PBS -l select=1:ncpus={}".format(ncpus))
if omp_threads:
file_stream.writelines(":ompthreads={}".format(ncpus))
file_stream.write(":mem=3gb\n")
file_stream.write("#PBS -l place={}\n".format(place))
file_stream.write("#PBS -q {}\n".format(queue))
file_stream.write("#PBS -N {}\n".format(item_type))
file_stream.write("\n")
file_stream.write("cd $PBS_O_WORKDIR\n")
file_stream.write("\n")
file_stream.write(". /usr/share/Modules/init/bash\n")
file_stream.write("\n")
file_stream.write(setCompilerVariables)
file_stream.write(setMklVariables)
file_stream.write(setMpiVariables)
file_stream.write("\n")
file_stream.write("time {}{} {}{}\n".format(command, executable, directory, examplesName))
file_stream.write("\n")
file_stream.write("qstat -f $PBS_JOBID\n")
file_stream.write("exit\n")
file_stream.close()
def build(self):
"""
call subprocess to build new release
:return:
"""
message(mode='INFO', text='Building {}'.format(self._item.configuration))
try:
Popen(self._subject.get_build_command(self._item), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
def run_item(self):
"""
call subprocess to run code for test item
:return:
"""
message(mode='INFO', text='Running {}'.format(self._item.name()))
try:
call(self._subject.get_execution_command(self._item), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
def pack_results(self):
"""
pack results on remote computer into tar file to download them later on
:return:
"""
if self._subject.location == 'remote':
message(mode='INFO', text='Pack results {}'.format(self._item.name()))
pack_tar_file(self._item.directory)
else:
message(mode='INFO', text='{} is local - Nothing done'.format(self._subject.computer))
def update_release(self):
"""
copy built files in special folder and rename them for release
generate new release folder if it does not exist
:return:
"""
message(mode='INFO', text='Updating release {} {}'.format(system(), self._item.configuration))
generate_folder(self._subject.directory + 'releases')
built_file = self._subject.get_built_file(self._item)
copy_file(built_file, self._subject.get_built_file_for_release(self._item))
def get_results(self):
"""
download and unpack tar file from remote to local computer,
if local computer is windows, convert files into dos-format
:return:
"""
if self._subject.location == 'remote':
message(mode='INFO', text='Get results {}'.format(self._item.name()))
mod = __import__(self._subject.computer)
generate_folder(self._item.directory)
if path.exists(self._item.directory):
# clear local directory
files = glob(self._item.directory + '*')
for file in files:
remove_file(file)
winscp_command = 'get {}results.tar {}'.format(
self._item.directory_computer_selected, self._item.directory)
call_winscp([winscp_command], self._subject.user, self._subject.hostname, mod.pwd, output_flag=True)
unpack_tar_file(self._item.directory) # tar file on local computer
if system() == 'Windows':
message(mode='INFO', text=' Convert to dos')
for file in listdir(self._item.directory):
unix2dos(file, output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
else:
message(mode='INFO', text='{} is local - Nothing done'.format(self._subject.computer))
def replace_nans_and_inds(self):
"""
replace each nan by 999 and remove each IND in all tec files
:return:
"""
message(mode='INFO', text='Replace nans and inds {}'.format(self._item.name()))
if path.exists(self._item.directory):
for file_name in listdir(self._item.directory):
if file_name.endswith('.tec'):
message(mode='INFO', text='File: {}'.format(file_name))
try:
infile = open(self._item.directory + file_name, 'r')
outfile = open('{}new_{}'.format(self._item.directory, file_name), 'w')
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
else:
for line in infile:
line = line.replace('nan', '999')
line = line.replace('#IND', '')
outfile.write(line)
infile.close()
outfile.close()
move_file('{}new_{}'.format(self._item.directory, file_name), self._item.directory + file_name)
else:
message(mode='ERROR', text='Directory missing')
def generate_jpgs(self):
"""
generate JPG with tecplot
1. generate tecplot macro
2. call tecplot via subprocess
3. remove tecplot macro
:return:
"""
message(mode='INFO', text='Generate Jpgs ' + self._item.type)
if path.exists(self._subject.directory_plot):
chdir(self._subject.directory_plot)
for file in glob("{}*".format(self._item.type)):
filename = path.splitext(file)[0]
message(mode='INFO', text=' {}'.format(filename))
write_tecplot_macro_for_jpg(self._subject.directory_plot, filename)
layout = '{}{}.lay'.format(self._subject.directory_plot, filename)
try:
call('{} {} -b -p {}_genJPG.mcr'.format(tecplot, layout, self._subject.directory_plot), shell=True)
except Exception as err:
message(mode='ERROR', text='{}'.format(err))
remove_file('{}_genJPG.mcr'.format(self._subject.directory_plot), output_flag=False)
else:
message(mode='ERROR', text='Directory missing')
def get_item(_id):
item = ItemModel.find_item_by_id(_id)
if item:
return jsonify(item.to_dict())
return message("There is no item whose id is equal {}".format(_id), 400)
def create_user():
new_user = UserModel.create_from_dict(g.data)
new_user.save_to_db()
return message("User {} was created".format(new_user.username), 201)
def get_store(_id):
store = StoreModel.find_store_by_id(_id)
if store:
return jsonify(store.to_dict())
return message("There is no store whose id is equal {}".format(_id), 400)
