def item_is_to_test(self, item_case): """ check test_mode, state, used for simulation, plotting operation type (case item exists) (all items (i.e. configurations) involved in building operation type) :param item_case: (string or None) :return: (bool )False: not involved, True: involved or ERROR """ execute_flag = True if location == 'local' and item_case: if self.__setting_inst.test_mode == '0': # no test mode - select everything by typing on console # and than the items in the loop (in environment run) is involved pass elif self.__setting_inst.test_mode == '1': # via browser execute_flag = self.__setting_inst.query_column_entry_for_name('cases', item_case, 'state') elif self.__setting_inst.test_mode == '2': # via CI tool if not self.__setting_inst.query_column_entry_for_name('cases', item_case, 'active'): execute_flag = False else: message(mode='ERROR', not_supported=self.__setting_inst.test_mode) # returns '1' return execute_flag # remote always '1'
def query_id_for_name(self, table, name): """ query an id from a table for a given name requirement: table columns must be: id, name, ... name must be unique in table (not checked) :param table: (string): name of table in SQL schema :param name: (string): entry in column name :return: id (string) or '-1' if no id found (exception) """ if not name: # empty string return None # no name given get_table_data_from_database else: # query_text = "SELECT {} FROM computer WHERE name=%s" # .format(table, name) # .encode(encoding='UTF-8', errors='strict') # query_text2 = query_text.decode(encoding='UTF-8', errors='strict') # .format(table, name) # row_list = self.query("SELECT id FROM " + str(table) + " WHERE name=%s", [name]) query_text = "SELECT id from {} WHERE name='{}'".format(table, name) row_list = self.query(query_text) if row_list[0] is not None: return str(row_list[0]['id']) # each row_list entry is dictionary {'id': (string)} else: message(mode='ERROR', text='Column id for {} not found in table {}'.format(name, table)) return '-1' # exception
def clear_folder(self): """ delete files 1. configurationShared.outputFileEndings (*.tec, *.txt, *.plt, *.vtk) 2. results.tar 3. configurationShared.outputFile (remote folder for remote computer) :return: """ message(mode='INFO', text='Clear simulation folder \n {}'.format(self._item.name())) # file_endings = list() # concerns files with outputFileEndings and *.plt for ending in outputFileEndings: file_endings.append(ending) file_endings.append('plt') if path.exists(self._item.directory): for file in listdir(self._item.directory): for ending in file_endings: if file.endswith('.{}'.format(ending)): remove_file(self._item.directory + file, False) tarfile_remote = '{}results.tar'.format(self._item.directory) remove_file(tarfile_remote, False) output_file = self._item.directory + outputFile remove_file(output_file, False) else: message(mode='ERROR', text='Directory missing')
def write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script): """ :param subject: (Subject) :param item: (Item) :param operation_type: (one-char string) :param operation: (one-char string) :param shell_script: (string) :return: """ try: f = open(shell_script, 'w') except Exception as err: message(mode='ERROR', text='{0}'.format(err)) else: f.write('#!/bin/sh\n') f.write('module load Python-3.3.2\n') f.write('python3 {}testingEnvironment/scripts/icbc/customized/run_remote.py '.format(subject.directory_root)) f.write('{} {} {} {} {} '.format(subject.computer, subject.user, subject.code, subject.branch, getpid())) if operation_type == 'b': # building f.write('None None {} None None '.format(item.configuration)) elif operation_type == 's': # simulating f.write('{} {} {} {} {} '.format( item.type, item.case, item.configuration, item.flow_process, item.element_type)) elif operation_type == 'p': # plotting f.write(' {} {} {} None None '.format(item.type, item.case, item.configuration)) f.write('{} {}'.format(operation_type, operation)) f.close()
def clear_folder(self): """ delete *.tec, *.txt, *.asc, *plt and results.tar (local folder for remote computer) :return: """ message(mode='INFO', text='Clear plotting folder ' + self._item.name()) # remove_file("{}results_{}.jpg".format(self._subject.directory_plot, self._item.type), False) file_endings = list() # concerns files with outputFileEndings and *.plt for ending in outputFileEndings: file_endings.append(ending) file_endings.append('plt') if path.exists(self._item.directory): for file in listdir(self._item.directory): for ending in file_endings: if file.endswith('.{}'.format(ending)): if self._subject.location == 'remote' or ending != 'tec': # do not delete tec if local remove_file(self._item.directory + file, False) # myLocalTarFile = self._item.directory + 'results.tar' # done in get_results() # if path.isfile(myLocalTarFile): # remove(myLocalTarFile) else: message(mode='ERROR', text='Directory missing')
def clear_folder(self): """ delete built files (from folder where they are after compilation) :return: """ message(mode='INFO', text='Removing release {} {}'.format(system(), self._item.configuration)) remove_file(self._subject.get_built_file(self._item), False)
def wait(self): """ wait until release exists :return: """ message(mode='INFO', text='Waiting for release {} {}'.format(system(), self._item.configuration)) wait_for_file(self._subject.get_built_file(self._item))
def write_winscp_file(file_winscp, winscp_command_list, user, hostname, password, output_flag=True): """ :param file_winscp: (string) :param winscp_command_list: (string list) :param user: (string) :param hostname: (string) :param password: (string) :param output_flag: (bool) :return: """ file_winscp = adapt_path(file_winscp) if output_flag: message(mode='INFO', text=' Write winscp file') try: f = open(file_winscp, 'w') except OSError as err: message(mode='ERROR', text='OS error: {0}'.format(err)) else: f.write('option batch abort \n') f.write('option confirm off \n') f.write('open sftp://{}:{}@{}/ \n'.format(user, password, hostname)) for command in winscp_command_list: f.write('{} \n'.format(command)) f.write('exit') f.close()
def delete_store(_id): store = StoreModel.find_store_by_id(_id) if store: name = store.name store.delete_from_db() return message('The {} was deleted from database'.format(name), 200) return message("There is no store whose id is equal {}".format(_id), 400)
def select_items_to_test(self, operation_type, operation, computer): """ generate examples Lists to loop over in Environment run :param operation_type: :param operation: :param computer: :return: """ if operation_type == 'b': # building (only configuration) self.__item_constituents.type_list = [None] self.__item_constituents.case_list = [[None]] self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test( level='configurations', computer_of_subject=computer) elif operation_type == 's': # simulating (all item constituents (type, case, configuration)) self.__item_constituents.type_list = self.select_constituent_of_items_to_test(level='types') self.__item_constituents.case_list = self.select_constituent_of_items_to_test(level='cases') self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test( level='configurations', computer_of_subject=computer) elif operation_type == 'p': # plotting self.__item_constituents.type_list = self.select_constituent_of_items_to_test(level='types') if operation == 'j' or operation == 'w': # generate or wait for JPG (only type) self.__item_constituents.case_list = [[None]] self.__item_constituents.configuration_list = [None] else: # all item constituents (type, case, configuration) self.__item_constituents.case_list = self.select_constituent_of_items_to_test(level='cases') self.__item_constituents.configuration_list = self.select_constituent_of_items_to_test( level='configurations', computer_of_subject=computer) else: message(mode='ERROR', not_supported='operation_type {}'.format(operation_type))
def select_constituent_of_items_to_test(self, level, computer_of_subject=None): """ calls member function selectNames to get item constituents if they are not previously selected (i.e. in environment constructor or anywhere in last operations) :param level: :param computer_of_subject: :return: list (string) for one item constituent (meaning: types, cases, configuration) nested list if table = 'cases' """ if level == 'types': if self.__item_constituents.type_list == [None]: # len(self.__item_constituents.type_list) == 0 or not self.__item_constituents.type_list[0]: return self.get_name_list(table='types') else: return self.__item_constituents.type_list elif level == 'cases': if self.__item_constituents.case_list == [[None]]: # len(self.__item_constituents.case_list) == 0 or self.__item_constituents.case_list[0] == [None]: return self.get_name_list(table='cases') else: return self.__item_constituents.case_list elif level == 'configurations': if self.__item_constituents.configuration_list == [None]: # len(self.__item_constituents.configuration_list) == 0 \ # or not self.__item_constituents.configuration_list[0]: return self.get_name_list(table='configurations', computer=computer_of_subject) else: return self.__item_constituents.configuration_list else: message(mode='ERROR', not_supported='level ' + level)
def id2name(self, table, id_selected, lists_inst): """ Convert id to name (list) and return this as a nested list [['...']] class member to have access to __selectedTypeIdList :param table: :param id_selected: :param lists_inst: :return: """ if id_selected == 'a': if table == 'cases': lists_inst.name.append(deepcopy(lists_inst.nameSub)) return lists_inst.name else: return lists_inst.nameSub elif id_selected == 'r': if table == 'types': for id_running in self.__selectedTypeIdList: lists_inst.name.append(deepcopy(lists_inst.nameSub[get_list_id(id_running, lists_inst.id)])) return lists_inst.name else: message(mode='ERROR', text='Range supported only for types') return None else: # single item lists_inst.name.append(deepcopy(lists_inst.nameSub[get_list_id(id_selected, lists_inst.id)])) if table == 'cases': name_list2 = list() name_list2.append(deepcopy(lists_inst.name)) return name_list2 else: return lists_inst.name
def write_num(self): """ call member of SimulationData to write file *.num :return: """ message(mode='INFO', text='Write *.num {}'.format(self._item.name())) self._simulation_data.write_num(self._item.directory)
def partition_mesh(self, directory): """ reloads prc.py to get parallelization data and writes *.pbs file :param directory: (string) :return: """ script_partition = "{}{}partition.sh".format(rootDirectory, adapt_path("testingEnvironment\\scripts\\")) chdir(directory) mesh_file = "{}{}.msh".format(directory, examplesName) if not path.isfile(mesh_file): # no file *.msh, than look if one remained from previous mesh partitioning mesh_file = "{}{}_mesh.txt".format(directory, examplesName) if path.isfile(mesh_file): try: if self.__processing.mode == "mpi_elements": # for OGS_FEM_MPI, ... Popen( script_partition + " {} -e -asci {}".format(self.__processing.number_cpus, directory), shell=True, ) if self.__processing.mode == "mpi_nodes": # for OGS_FEM_PETSC Popen( "{} {} -n -binary {}".format(script_partition, self.__processing.number_cpus, directory), shell=True, ) except Exception as err: message(mode="ERROR", text="{}".format(err)) else: message(mode="ERROR", text="Mesh file missing")
def wait(self): """ wait until jpg file is in folder :return: """ message(mode='INFO', text='Waiting for jpg file {}'.format(self._item.type)) wait_for_file("{}results_{}.jpg".format(self._subject.directory_plot, self._item.type))
def write_shell_script_for_remote_run(subject, item, operation_type, operation, shell_script): """ :param subject: (Subject) :param item: (Item) :param operation_type: (one-char string) :param operation: (one-char string) :param shell_script: (string) :return: """ try: f = open(shell_script, 'w') except Exception as err: message(mode='ERROR', text='{0}'.format(err)) else: f.write('#!/bin/sh\n') f.write('module load python3.3\n') f.write('python ' + subject.directory_root + 'testingEnvironment/scripts/icbc/customized/run_remote.py ') f.write(subject.computer + ' ' + subject.user + ' ' + subject.code + ' ' + subject.branch + ' ') if operation_type == 'b': # building f.write('No No ' + item.configuration + ' ') else: f.write(item.type + ' ' + item.case + ' ' + item.configuration + ' ') f.write(operation_type + ' ' + operation) f.close()
def get_processing_data_from_module(self, id_local_process): """ get processing (parallelization) data from module function is run on remote computer module name contain process id from local computer for its identification restriction: modules are not reloaded since only one operation on remote computer :return: """ try: processing = __import__("processing_" + id_local_process) self.__processing.number_cpus = processing.number_cpus self.__processing.mode = processing.mode directory_temp = adapt_path("testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory)) remove_file(directory_temp + "processing_{}.py".format(id_local_process), False) except Exception as err: message(mode="ERROR", text="OS error: {}".format(err)) # remove byte code if it exists try: remove_file(directory_temp + "processing_{}.pyc".format(id_local_process), False) except: pass
def get_num_data_from_modules(self, id_local_process): """ get numerics data global, flow, mass, heat function is run on remote computer module names contain process id from local computer for its identification delete modules after data obtained restriction: modules are not reloaded since only one operation on remote computer :return: """ try: directory_temp = adapt_path("{}testingEnvironment\\scripts\icbc\\temp\\".format(rootDirectory)) # first global to get process flags self.get_num_data_for_process("global", id_local_process, directory_temp) # processes self.get_num_data_for_process("flow", id_local_process, directory_temp) if self.__numerics_global.processes.mass_flag: self.get_num_data_for_process("mass", id_local_process, directory_temp) if self.__numerics_global.processes.heat_flag: self.get_num_data_for_process("heat", id_local_process, directory_temp) if self.__numerics_global.processes.deformation_flag: self.get_num_data_for_process("deformation", id_local_process, directory_temp) if self.__numerics_global.processes.fluid_momentum_flag: self.get_num_data_for_process("fluid_momentum", id_local_process, directory_temp) if self.__numerics_global.processes.overland_flag: self.get_num_data_for_process("overland", id_local_process, directory_temp) except Exception as err: message(mode="ERROR", text="OS error: {0}".format(err))
def delete_item(_id): item = ItemModel.find_item_by_id(_id) if item: name = item.name item.delete_from_db() return message('The {} was deleted from database'.format(name), 200) return message("There is no item whose id is equal {}".format(_id), 400)
def write_num(self, directory): """ Reads global numerics and parallelization data from files and writes *.num file for simulation :param directory: :return: """ try: file_stream = open(directory + examplesName + ".num", "w") except OSError as err: message(mode="ERROR", text="OS error: {0}".format(err)) else: if self.__numerics_global.coupled_flag: file_stream.write("\n$OVERALL_COUPLING\n") file_stream.write(" {} {}\n".format(coupling_iterations_min, coupling_iterations_max)) self.write_process_into_numerics_file(file_stream, "flow") if self.__numerics_global.processes.mass_flag: self.write_process_into_numerics_file(file_stream, "mass") if self.__numerics_global.processes.heat_flag: self.write_process_into_numerics_file(file_stream, "heat") if self.__numerics_global.processes.deformation_flag: self.write_process_into_numerics_file(file_stream, "deformation") if self.__numerics_global.processes.fluid_momentum_flag: self.write_process_into_numerics_file(file_stream, "fluid_momentum") if self.__numerics_global.processes.overland_flag: self.write_process_into_numerics_file(file_stream, "overland") file_stream.write("\n#STOP\n") file_stream.close()
def query_ids_from_column_entries(self, table, column_name, entry): """ ... :param table: (string): name of table in SQL schema :param item_id: (string): entry in id column (primary key) :param column_name: (string) name of column where entry is searched from :param entry: (string) :return: (string) column entry, '-1' if no entry found (exception) """ result_list = list() row_list = self.query("SELECT t.id FROM {} t WHERE t.{}={}".format(table, column_name, entry)) try: a = row_list[0] except: message(mode='ERROR', text='Column entry {} of {} not found'.format(entry, column_name)) return '-1' # exception else: i = 0 while True: try: a = row_list[i] except: break else: result_list.append(row_list[i]['id']) i += 1 return result_list
def partition_mesh(self): """ call member of SimulationData to partition shell script :return: """ message(mode='INFO', text='Mesh partition {}'.format(self._item.name())) self._simulation_data.partition_mesh(self._item.directory)
def write_pbs(self): """ call member of SimulationData to write file run.pbs :return: """ message(mode='INFO', text='Write run.pbs {}'.format(self._item.name())) self._simulation_data.write_pbs(self._item.directory, self._subject.get_built_file(self._item), self._item.type)
def wait(self): """ Wait until configurationShared.outputFile exists :return: """ message(mode='INFO', text='Waiting for output file {}'.format(self._item.name())) # wait_for_file(self._item.directory + outputFile)
def write_pbs(self, directory, executable, item_type): """ reloads prc.py to get parallelization data and writes *.pbs file :param directory: :param executable: :param item_type: :return: """ # config omp_threads = None if self.__processing.mode == "sequential": ncpus = "1" command = " " place = "group=host" elif self.__processing.mode == "omp": ncpus = self.__processing.number_cpus omp_threads = self.__processing.number_cpus command = " " place = "group=host" elif self.__processing.mode == "mpi_elements" or self.__processing.mode == "mpi_nodes": # parallel ncpus = self.__processing.number_cpus command = "mpirun -r rsh -machinefile $PBS_NODEFILE -n " + ncpus + " " place = "scatter" else: message(mode="ERROR", not_supported="Mode {}".format(self.__processing.mode)) return # write try: file_stream = open("{}run.pbs".format(directory), "w") except OSError as err: message(mode="ERROR", text="OS error: {}".format(err)) else: file_stream.write("#!/bin/bash\n") file_stream.write("#PBS -o {}screenout.txt\n".format(directory)) file_stream.write("#PBS -j oe\n") file_stream.write("#PBS -r n\n") file_stream.write("#PBS -l walltime={}\n".format(walltime)) file_stream.write("#PBS -l select=1:ncpus={}".format(ncpus)) if omp_threads: file_stream.writelines(":ompthreads={}".format(ncpus)) file_stream.write(":mem=3gb\n") file_stream.write("#PBS -l place={}\n".format(place)) file_stream.write("#PBS -q {}\n".format(queue)) file_stream.write("#PBS -N {}\n".format(item_type)) file_stream.write("\n") file_stream.write("cd $PBS_O_WORKDIR\n") file_stream.write("\n") file_stream.write(". /usr/share/Modules/init/bash\n") file_stream.write("\n") file_stream.write(setCompilerVariables) file_stream.write(setMklVariables) file_stream.write(setMpiVariables) file_stream.write("\n") file_stream.write("time {}{} {}{}\n".format(command, executable, directory, examplesName)) file_stream.write("\n") file_stream.write("qstat -f $PBS_JOBID\n") file_stream.write("exit\n") file_stream.close()
def build(self): """ call subprocess to build new release :return: """ message(mode='INFO', text='Building {}'.format(self._item.configuration)) try: Popen(self._subject.get_build_command(self._item), shell=True) except Exception as err: message(mode='ERROR', text='{}'.format(err))
def run_item(self): """ call subprocess to run code for test item :return: """ message(mode='INFO', text='Running {}'.format(self._item.name())) try: call(self._subject.get_execution_command(self._item), shell=True) except Exception as err: message(mode='ERROR', text='{}'.format(err))
def pack_results(self): """ pack results on remote computer into tar file to download them later on :return: """ if self._subject.location == 'remote': message(mode='INFO', text='Pack results {}'.format(self._item.name())) pack_tar_file(self._item.directory) else: message(mode='INFO', text='{} is local - Nothing done'.format(self._subject.computer))
def update_release(self): """ copy built files in special folder and rename them for release generate new release folder if it does not exist :return: """ message(mode='INFO', text='Updating release {} {}'.format(system(), self._item.configuration)) generate_folder(self._subject.directory + 'releases') built_file = self._subject.get_built_file(self._item) copy_file(built_file, self._subject.get_built_file_for_release(self._item))
def get_results(self): """ download and unpack tar file from remote to local computer, if local computer is windows, convert files into dos-format :return: """ if self._subject.location == 'remote': message(mode='INFO', text='Get results {}'.format(self._item.name())) mod = __import__(self._subject.computer) generate_folder(self._item.directory) if path.exists(self._item.directory): # clear local directory files = glob(self._item.directory + '*') for file in files: remove_file(file) winscp_command = 'get {}results.tar {}'.format( self._item.directory_computer_selected, self._item.directory) call_winscp([winscp_command], self._subject.user, self._subject.hostname, mod.pwd, output_flag=True) unpack_tar_file(self._item.directory) # tar file on local computer if system() == 'Windows': message(mode='INFO', text=' Convert to dos') for file in listdir(self._item.directory): unix2dos(file, output_flag=False) else: message(mode='ERROR', text='Directory missing') else: message(mode='INFO', text='{} is local - Nothing done'.format(self._subject.computer))
def replace_nans_and_inds(self): """ replace each nan by 999 and remove each IND in all tec files :return: """ message(mode='INFO', text='Replace nans and inds {}'.format(self._item.name())) if path.exists(self._item.directory): for file_name in listdir(self._item.directory): if file_name.endswith('.tec'): message(mode='INFO', text='File: {}'.format(file_name)) try: infile = open(self._item.directory + file_name, 'r') outfile = open('{}new_{}'.format(self._item.directory, file_name), 'w') except Exception as err: message(mode='ERROR', text='{}'.format(err)) else: for line in infile: line = line.replace('nan', '999') line = line.replace('#IND', '') outfile.write(line) infile.close() outfile.close() move_file('{}new_{}'.format(self._item.directory, file_name), self._item.directory + file_name) else: message(mode='ERROR', text='Directory missing')
def generate_jpgs(self): """ generate JPG with tecplot 1. generate tecplot macro 2. call tecplot via subprocess 3. remove tecplot macro :return: """ message(mode='INFO', text='Generate Jpgs ' + self._item.type) if path.exists(self._subject.directory_plot): chdir(self._subject.directory_plot) for file in glob("{}*".format(self._item.type)): filename = path.splitext(file)[0] message(mode='INFO', text=' {}'.format(filename)) write_tecplot_macro_for_jpg(self._subject.directory_plot, filename) layout = '{}{}.lay'.format(self._subject.directory_plot, filename) try: call('{} {} -b -p {}_genJPG.mcr'.format(tecplot, layout, self._subject.directory_plot), shell=True) except Exception as err: message(mode='ERROR', text='{}'.format(err)) remove_file('{}_genJPG.mcr'.format(self._subject.directory_plot), output_flag=False) else: message(mode='ERROR', text='Directory missing')
def get_item(_id): item = ItemModel.find_item_by_id(_id) if item: return jsonify(item.to_dict()) return message("There is no item whose id is equal {}".format(_id), 400)
def create_user(): new_user = UserModel.create_from_dict(g.data) new_user.save_to_db() return message("User {} was created".format(new_user.username), 201)
def get_store(_id): store = StoreModel.find_store_by_id(_id) if store: return jsonify(store.to_dict()) return message("There is no store whose id is equal {}".format(_id), 400)