def generate_doxygen(doxydoc, verbose): """ Generate the Doxygen documentation (In html) for the Python and the sources @param doxydoc name of the doxygen folder to use (doxydocs or doxypydocs) @param verbose If True display doxygen listing """ if doxydoc == "doxydocs": # Checking that the converter is compiled converter_path = path.join(CFGS.get_root(), 'optionals', 'addons', 'to_f90') if not path.exists(path.join(converter_path, 'f77_to_f90')): raise TelemacException( "You need to compile the converter in :\n"+converter_path) # Compiling sources doxygen doxy_dir = path.join(CFGS.get_root(), 'documentation', doxydoc) doxy_file = path.join(doxy_dir, 'Doxyfile') compile_doxygen(doxy_file, verbose)
def get_report_path(report_name, type_valid): """ Build report full path $HOMETEL/[report_name]_[config]_[version]_[type_valid]_[date].csv where: - report_name is the one given as argument - config is the name of the configuration for which the validation is run - version is the version of the code - type_valid is the one given as argument - date is the data at which the report is written @param report_name (str) Name given to the report @param type_valid (str) Type of validation (notebook, examples...) @returns (str) The path """ full_report_name = "{}_{}_{}_{}_{}.csv".format(\ report_name, CFGS.cfgname, CFGS.configs[CFGS.cfgname].get('version', 'trunk'), type_valid, ttime.strftime("%Y-%m-%d-%Hh%Mmin%Ss", ttime.localtime(ttime.time()))) report_path = path.join(CFGS.get_root(), full_report_name) return report_path
def hide_root(string): """ Replace the path to sources of telemac by <root> @param string (string) The string in which to hide root @return (string) The updated string """ return string.replace(CFGS.get_root(), "<root>")
def run_validation_notebooks(options, report, xcpts): """ Run validation of the notebooks @param options (ArgumentParser) Options of the script @param report (Report) Contains execution time @param xcpts (Message) Error handler """ root = CFGS.get_root() if options.args != []: # Files to run given in arguments nb_files = options.args else: # Looking through notebook folder for notebook to run nb_files = [] for dirpath, _, ffiles in walk(path.join(root, 'notebooks')): for ffile in ffiles: # Skipping jupyter temporary folders if '.ipynb' in dirpath: continue # If we have a notebook if '.ipynb' in ffile: nb_files.append(path.join(dirpath, ffile)) # Removing exlcuded notebooks if options.nb_exclude is not None: options.nb_exclude = options.nb_exclude.strip("'") for nb_file in list(nb_files): for exclude in options.nb_exclude.split(','): if nb_file.endswith(exclude + ".ipynb"): if nb_file in nb_files: print(" ~> Excluding: ", nb_file) nb_files.remove(nb_file) # Run notebook validation n_nb = len(nb_files) for i, nb_file in enumerate(sorted(nb_files)): print('Validation <{}/{}> ~> Running notebook {} in {}'\ .format(i+1, n_nb, path.basename(nb_file), path.dirname(nb_file))) try: start = time.time() run_notebook(nb_file, options.nb_timeout, update_nb=options.nb_update) end = time.time() report.add_notebook(nb_file, end - start, True) except Exception as exc: if options.bypass: report.add_notebook(nb_file, 0.0, False) xcpts.add_messages([{'name': nb_file, 'msg': str(exc)}]) else: raise exc
def get_dico(module): """ Returns path of the dictionary for a given module @param module (str) name of a telemac-mascaret module @returns the path """ from config import CFGS if CFGS is None: raise TelemacException(\ "This function only wors if a configuration is set") return path.join(CFGS.get_root(), 'sources', module, module+'.dico')
def generate_doxygen(doxydoc, verbose): """ Generate the Doxygen documentation (In html) for the Python and the sources @param doxydoc name of the doxygen folder to use (doxydocs or doxypydocs) @param verbose If True display doxygen listing """ # Compiling sources doxygen doxy_dir = path.join(CFGS.get_root(), 'documentation', doxydoc) doxy_file = path.join(doxy_dir, 'Doxyfile') if doxydoc == "doxypydocs": try: import doxypypy except ImportError: raise TelemacException("doxypypy is mandatory to compile doxygen " "for Telemac scripts") compile_doxygen(doxy_file, verbose)
def run_validation_python(cfg, options, report, xcpts): """ Run validation for vnv Python scripts @param cfg (Dict) Configuration information @parma options (ArgumentParser) List of arguments @param report (Report) Time of actions @param xcpts () Error handler """ # Building list of files to run if options.args != []: list_files = [] for ffile in options.args: list_files.append(path.abspath(ffile)) else: # Looping on all folders to find the scripts to execute list_files = [] # Loop on modules for code_name in cfg['VALIDATION']: val_root = cfg['val_root'] dirpath, dirnames, _ = next(walk(path.join(val_root, code_name))) for ddir in dirnames: _, _, filenames = next(walk(path.join(dirpath, ddir))) for fle in filenames: root, ext = path.splitext(path.basename(fle)) if ext == '.py' and root[0:4] == 'vnv_': # check rank and tag file_name = path.join(dirpath, ddir, fle) if check_python_rank_tags(file_name, options): list_files.append(file_name) n_files = len(list_files) root = CFGS.get_root() for ifile, py_file in enumerate(sorted(list_files)): if options.cleanup or options.full_cleanup: clean_vnv_working_dir(py_file, full=options.full_cleanup) else: print('\n\nValidation < {}/{} > of {}'\ .format(ifile+1, n_files, py_file.replace(root, '<root>'))) run_python(py_file, options, report, xcpts)
def run_validation_python_mpi(cfg, options, report, xcpts): """ Run validation for vnv Python scripts Normale mode @param cfg (Dict) Configuration information @param options (ArgumentParser) List of arguments @param report (Report) Time of actions @param xcpts () Error handler """ # Building list of files to run list_files = get_list_python_files(cfg, options) n_files = len(list_files) root = CFGS.get_root() for ifile, py_file in enumerate(sorted(list_files)): if options.cleanup or options.full_cleanup: clean_vnv_working_dir(py_file, full=options.full_cleanup) else: print('\n\nValidation < {}/{} > of {}'\ .format(ifile+1, n_files, py_file.replace(root, '<root>'))) run_python(py_file, options, report, xcpts)
def main(): """ Main function of gretel """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + 72 * '~' + '\n') parser = argparse.ArgumentParser(\ description='Run the merging step (gretel)') parser = add_config_argument(parser) parser.add_argument(\ "--input-file", dest="input_file", default='', help="Name of gretel parameter file (GRETEL.PAR). "\ "This option will surcharge all the others") parser.add_argument(\ "--geo-file", dest="geo_file", default='T2DGEO', help="Name of the geometry file associated with the "\ "file to be merged") parser.add_argument(\ "--geo-file-format", dest="geo_file_fmt", default='SERAFIN', help="Format of the geometry file(SERAFIN,SERAFIND or MED), "\ "default is SERAFIN") parser.add_argument(\ "--res-file", dest="res_file", default='T2DRES', help="Name of the file to be merged") parser.add_argument(\ "--res-file-format", dest="res_file_fmt", default='SERAFIN', help="Format of the geometry file(SERAFIN,SERAFIND or MED), "\ "default is SERAFIN") parser.add_argument(\ "--bnd-file", dest="bnd_file", default='T2DCLI', help="Name of the boundary file") parser.add_argument(\ "--ncsize", dest="ncsize", default=8, help="Number of partitions (should be equal to number of "\ "parallel processors), default is 8") parser.add_argument(\ "--nplan", dest="nplan", default=0, help="Number of horizontal levels ,default is 0") args = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(args) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ CFGS.compute_execution_info() if args.input_file != "": with open(args.input_file, 'r') as f: geo_file = f.readline().strip('\n') geo_file_fmt = f.readline().strip('\n') bnd = f.readline().strip('\n') res_file = f.readline().strip('\n') res_file_fmt = f.readline().strip('\n') ncsize = f.readline().strip('\n') nplan = f.readline().strip('\n') else: geo_file = args.geo_file geo_file_fmt = args.geo_file_fmt bnd = args.bnd_file res_file = args.res_file res_file_fmt = args.res_file_fmt ncsize = args.ncsize nplan = args.nplan # Getting partel command from configuration pbin = path.join(CFGS.get_root(), 'builds', CFGS.cfgname, 'bin') grecmd = get_gretel_cmd(pbin, CFGS.configs[CFGS.cfgname]) # Running paritionning run_gretel(grecmd, res_file, res_file_fmt, geo_file, geo_file_fmt, bnd, ncsize, nplan, False) print('\n\nMy work is done\n\n') sys.exit(0)
def main(): """ Main program for the compilation of the documentation of the telemac-mascaret system """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n'+'~'*72+'\n') parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, description=('''\n By Default all the documentation are generated\n use the options --validation/reference/user/release/theory to compile only one ''')) parser = add_config_argument(parser) parser.add_argument( "-v", "--verbose", action="store_true", dest="verbose", default=False, help="Will display listing for all commands") parser.add_argument( "-m", "--modules", dest="modules", default='', help="specify the list modules (, separated), default is all of them") parser.add_argument( "-M", "--misc", dest="misc", default='', help="specify the list of misc documentation (, separated) to compile, " "default is all of them") parser.add_argument( "--validation", action="store_true", dest="validation", default=False, help="Will generate the validation documentation") parser.add_argument( "--case-list", dest="case_list", default='', help="List of cas to include in the validation documentation" "separated by ',' (default all of them)") parser.add_argument( "--reference", action="store_true", dest="reference", default=False, help="Will generate the reference documentation") parser.add_argument( "--user", action="store_true", dest="user", default=False, help="Will generate the user documentation") parser.add_argument( "--release", action="store_true", dest="release_note", default=False, help="Will generate the release note") parser.add_argument( "--theory", action="store_true", dest="theory_guide", default=False, help="Will generate the theory guide") parser.add_argument( "--clean", action="store_true", dest="cleanup", default=False, help="Will remove all temporary file generated by pdflatex") parser.add_argument( "--fullclean", action="store_true", dest="fullcleanup", default=False, help="Same as clean but removes the pdf as well") # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ options = parser.parse_args() update_config(options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Compile the valiation documentation doall = not (options.validation or options.user or options.reference or options.release_note or options.theory_guide) cfg = CFGS.configs[CFGS.cfgname] # still in lower case root = CFGS.get_root() # Get what i to be compiled # By default everything if something is defined compiling only that if options.modules != '': module_list = options.modules.split(',') else: # all modules module_list = ['artemis', 'stbtel', 'sisyphe', 'postel3d', 'telemac2d', 'telemac3d', 'tomawac', 'waqtel', 'telapy', 'mascaret', 'gaia', 'nestor', 'khione'] if options.misc != '': misc_list = options.misc.split(',') module_list = [] else: # all docs misc_list = ['developer_guide', 'software_quality_plan', 'TelemacDocTemplate', 'git_guide', 'doxypydocs'] # If a module was specified or a specific documentation for modules # not compiling Misc documentation if options.modules != '' or not doall: misc_list = [] CFGS.compute_vnv_info() # Get version in config if it exist use trunk otherwise version = cfg.get('version', 'trunk') # Initialise output message output_mess = '\n\n' # Look on all the modules for the documentation for code_name in module_list: print('\nCompilation of the documentation for ' + code_name + '\n'+'~'*72) # list of what to do for the module todo = [] if options.validation or doall: if code_name not in ['telapy', 'mascaret']: # Building Validation LaTeX file doc_dir = path.join(root, 'documentation', code_name, 'validation') chdir(doc_dir) if options.case_list != '': list_of_case = options.case_list.split(',') else: list_of_case = list(cfg['VALIDATION'][code_name].keys()) list_of_case.remove('path') skiped_case = \ create_case_list_file( doc_dir, cfg['VALIDATION'][code_name]['path'], list_of_case, options.cleanup or options.fullcleanup) for case in skiped_case: output_mess += r' - /!\ Missing LaTeX file for ' + \ case+'\n' todo.append('validation') if options.reference or doall: if code_name not in ['telapy', 'mascaret', 'nestor']: # Path to the dictionary dictionary = path.join(root, 'sources', code_name, code_name+'.dico') # Path to latex File latex_file = path.join(root, 'documentation', code_name, 'reference', 'latex', 'Corpus.tex') # English only for now lng = '2' # Path to bin directory exe_path = path.join(\ root, 'builds', CFGS.cfgname, 'bin', 'damocles'+cfg['SYSTEM']['sfx_exe']) generate_ref_from_dict(\ exe_path, dictionary, latex_file, lng, options.cleanup or options.fullcleanup, options.verbose) todo.append('reference') if options.user or doall: if code_name not in ['mascaret']: # Normal Compilation of a LaTeX file todo.append('user') if options.theory_guide or doall: # theory guide only available for telemac3d if code_name in ['telemac3d', 'mascaret', 'waqtel']: todo.append('theory_guide') for doc_type in todo: print('\n ~> Compilation of the {} documentation'\ .format(doc_type)) doc_dir = path.join(root, 'documentation', code_name, doc_type) chdir(doc_dir) # Check if the file exist if path.exists(path.join(doc_dir, code_name + "_" + doc_type + ".tex")): compile_doc(doc_dir, code_name+'_'+doc_type, version, options.cleanup, options.fullcleanup, options.verbose) else: raise TelemacException(\ " - Error for {} {}, {}.tex " "not found ".format(code_name, path.basename(doc_dir), code_name+"_"+doc_type)) if not (options.cleanup or options.fullcleanup): output_mess += ' - Created %s_%s_%s.pdf\n' % \ (code_name, doc_type, version) # List of the other documentation print('\nCompilation of the documentation for Misc' + '\n'+'~'*72) for doc in misc_list: print('\n ~> Compilation of the {} documentation'.format(doc)) doc_dir = path.join(root, 'documentation', 'Misc', doc) if doc == 'notebook': notebook_dir = path.join(root, 'notebooks') generate_notebook_pdf(doc_dir, notebook_dir) elif doc in ['doxydocs', 'doxypydocs']: generate_doxygen(doc, options.verbose) else: chdir(doc_dir) if path.exists(path.join(doc_dir, doc + ".tex")): compile_doc(doc_dir, doc, version, options.cleanup, options.fullcleanup, options.verbose) else: raise TelemacException(\ " - Error in {}, {}.tex " "not found ".format(path.basename(doc_dir), doc)) if not (options.cleanup or options.fullcleanup) or \ doc not in ['notebooks', 'doxydocs', 'doxypydocs']: output_mess += ' - Created %s_%s.pdf\n' % \ (doc, version) print(output_mess) print('\n\n'+'~'*72) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nMy work is done\n\n') sys.exit(0)
def main(modules, example, nncsize, clean): """ Main function """ # Running main function root_dir = CFGS.get_root() if path.exists('ValidationTelApy.log'): remove('ValidationTelApy.log') fichier = open('ValidationTelApy.log', 'a') fichier.write("-----Listing Validation telapy-------\n") seq_only = {} skip_test = {} # Specifcation for each module for module in MODULE_HANDLED: seq_only[module] = [] skip_test[module] = [] # Sequential only test cases seq_only['telemac2d'].append('t2d_hydraulic_jump_v1p0.cas') seq_only['telemac2d'].append('t2d_hydraulic_jump_v2p0.cas') seq_only['telemac2d'].append('t2d_wesel.cas') seq_only['telemac2d'].append('t2d_wesel_pos.cas') seq_only['telemac2d'].append('t2d_delwaq.cas') seq_only['telemac2d'].append('t2d_ruptmoui.cas') seq_only['telemac2d'].append('t2d_triangular_shelf.cas') seq_only['telemac2d'].append('t2d_island.cas') seq_only['telemac2d'].append('t2d_tide-jmj_real_gen.cas') seq_only['telemac2d'].append('t2d_tide-jmj_type_gen.cas') seq_only['telemac2d'].append('t2d_dambreak_v1p0.cas') seq_only['telemac3d'].append('t3d_delwaq.cas') seq_only['telemac3d'].append('t3d_pluie.cas') seq_only['telemac3d'].append('t3d_tide-jmj_real_gen.cas') seq_only['artemis'].append('none') seq_only['tomawac'].append('tom_turning_wind.cas') seq_only['tomawac'].append('tom_manche.cas') seq_only['tomawac'].append('tom_manchelim.cas') # Test case that can not work with api # Using homere_adj not handle by api skip_test['telemac2d'].append('estimation') # Reruning telemac from homere not handled by api skip_test['telemac2d'].append('convergence') # Case that are not run by validation skip_test['telemac2d'].append('t2d_tide-jmj_type_med.cas') skip_test['telemac2d'].append('t2d_tide-ES_real.cas') # Non telemac3d case in folder skip_test['telemac3d'].append('t2d_canal.cas') skip_test['telemac3d'].append('p3d_amr.cas') skip_test['telemac3d'].append('p3d_bump.cas') skip_test['telemac3d'].append('p3d_canal.cas') skip_test['telemac3d'].append('p3d_cooper.cas') skip_test['telemac3d'].append('p3d_depot.cas') skip_test['telemac3d'].append('p3d_flume_slope.cas') skip_test['telemac3d'].append('p3d_gouttedo.cas') skip_test['telemac3d'].append('p3d_lock-hydro.cas') skip_test['telemac3d'].append('p3d_lock-nonhydro.cas') skip_test['telemac3d'].append('p3d_nonlinearwave.cas') skip_test['telemac3d'].append('p3d_piledepon.cas') skip_test['telemac3d'].append('p3d_piledepon-nonhydro.cas') skip_test['telemac3d'].append('p3d_pluie.cas') skip_test['telemac3d'].append('p3d_rouse.cas') skip_test['telemac3d'].append('p3d_stratification.cas') skip_test['telemac3d'].append('p3d_tetra.cas') skip_test['telemac3d'].append('p3d_vent.cas') skip_test['telemac3d'].append('p3d_V.cas') # Coupling test case skip_test['telemac3d'].append('depot') skip_test['telemac3d'].append('heat_exchange') # Artemis animated test case skip_test['artemis'].append('art_bj78_animated.cas') skip_test['artemis'].append('art_creocean_animated.cas') skip_test['artemis'].append('art_creocean_2.cas') skip_test['artemis'].append('art_creocean.cas') # Tomawac coupled test cases skip_test['tomawac'].append('3Dcoupling') for module in modules: fichier.write("-- For module " + module + "\n") module_dir = path.join(root_dir, 'examples', module) list_test_case = [] if example != '': list_test_case.append(example) else: list_test_case = sorted(listdir(module_dir)) # Sequential only test_case for i, test_case in enumerate(list_test_case): if test_case in skip_test[module]: continue case_dir = path.join(module_dir, test_case) tmp_dir = path.join(case_dir, 'tmp') print("<" + str(i + 1) + "/" + str(len(list_test_case)) + '> ' + str(test_case)) fichier.write('Running test case ' + test_case + '\n') list_file = copy_file_to_tmp.copy_file_to_tmp(\ case_dir, tmp_dir, \ module, root_dir, skip_test[module]) chdir(tmp_dir) for cas, fortran in list_file: # # Running Telemac based on telapy # if cas in skip_test[module]: continue # Get results names res_file = get_result_file_name.get_result_file_name( module, cas) api_res_file = res_file + '_api' # Running in sequential mode # if the case does not run in parallel if cas in seq_only[module]: ncsize = 1 else: ncsize = nncsize passed_api = run_telemac_api.run_telemac_api( module, cas, ncsize, fortran) if passed_api: shutil.move(res_file, api_res_file) # Running Telemac classical way # passed_normal = run_telemac_normal.run_telemac_normal( module, cas, ncsize) # # Result comparison between api and # classical Telemac computation # if not passed_normal: fichier.write(' Normal run crashed\n') if not passed_api: fichier.write(' Api run crashed\n') if not passed_api or not passed_normal: fichier.write( str(cas) + ' FAILED' + '\n') continue if not path.exists(res_file): fichier.write(' Missing ' + res_file + "\n") fichier.write( str(cas) + ' FAILED' + '\n') continue if not path.exists(api_res_file): fichier.write(' Missing ' + api_res_file + "\n") fichier.write( str(cas) + ' FAILED' + '\n') continue compare = cmp(res_file, api_res_file) if compare: fichier.write( str(cas) + ' PASSED' + '\n') else: fichier.write( str(cas) + ' FAILED' + '\n') if clean: chdir(module_dir + sep + test_case) shutil.rmtree(module_dir + sep + test_case + sep + 'tmp') fichier.write('my work is done ' + '\n')
def main(): """ Main function of partel.py """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + 72 * '~' + '\n') parser = argparse.ArgumentParser( description='Run the partitionning step (partel)') parser = add_config_argument(parser) parser.add_argument("--input-file", dest="input_file", default='', help="Name of partel parameter file (PARTEL.PAR)." " This option will surcharge all the others") parser.add_argument("--file", dest="geo_file", default='T2DGEO', help="Name of the file to be partitionned") parser.add_argument( "--file-format", dest="geo_file_fmt", default='SERAFIN', help="Format of the geometry file(SERAFIN,SERAFIND or MED), " "default is SERAFIN") parser.add_argument( "--bnd-file", dest="bnd_file", default='T2DCLI', help="Name of the boundary file associated to the mesh file, " "default is T2DCLI") parser.add_argument( "--ncsize", dest="ncsize", default=8, help="Number of partitions (should be equal to number of " "parallel processors), default is 8") parser.add_argument( "--section-name", dest="section_file", default='', help="Name of the section file, default no section file") parser.add_argument("--zone-name", dest="zone_file", default='', help="Name of the zone file, default no zone file") parser.add_argument("--weir-name", dest="weir_file", default='', help="Name of the weir file, default no weir file") parser.add_argument( "--partitioning-method", dest="part_method", default=1, help="Method used for the partitionning (1:metis, 2:scotch)") parser.add_argument("--concat", dest="concat", action="store_true", default=False, help="If true concatenate partel output") parser.add_argument( "--mpi", dest="mpi", action="store_true", default=False, help="Run partel as executable (note using command given in systel.cfg)" ) args = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(args) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ CFGS.compute_execution_info() if args.input_file != "": with open(args.input_file, 'r') as f: geo_file = f.readline().strip('\n') geo_file_fmt = f.readline().strip('\n') bnd_file = f.readline().strip('\n') ncsize = f.readline().strip('\n') part_method = f.readline().strip('\n') section_file = f.readline().strip('\n') zone_file = f.readline().strip('\n') weir_file = f.readline().strip('\n') _ = f.readline().strip('\n') _ = f.readline().strip('\n') concat = f.readline().strip('\n') else: concat = 'YES' if args.concat else 'NO' geo_file = args.geo_file geo_file_fmt = args.geo_file_fmt bnd_file = args.bnd_file ncsize = args.ncsize section_file = args.section_file zone_file = args.zone_file weir_file = args.weir_file part_method = args.part_method # Getting partel command from configuration pbin = path.join(CFGS.get_root(), 'builds', CFGS.cfgname, 'bin') if args.mpi: exe_ext = CFGS.configs[CFGS.cfgname]['SYSTEM']['sfx_exe'] parcmd = path.join(pbin, 'partel'+exe_ext+\ ' < <partel.par> >> <partel.log>') else: parcmd = get_partel_cmd(pbin, CFGS.configs[CFGS.cfgname], '') # Running paritionning run_partel(parcmd, geo_file, geo_file_fmt, bnd_file, ncsize, False, section_file, zone_file, weir_file, geo_file, geo_file_fmt, part_method, concat) print('\n\nMy work is done\n\n') sys.exit(0)
def main(module=None): """ @brief Main function of the runcode.py module @param module (string): the name of the module to run ( available modules are: telemac2d, telemac3d, artemis, tomawac, sisyphe, artemis, postel3d, ...) @return None """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + 72 * '~' + '\n') if module is None: parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, description=('''\n runcode is the execution launcher for all TELEMAC modules.\n where module can be:\n mascaret the 1D hydrodyanmic / tracer / water quality solver telemac2d the 2D hydrodyanmic / tracer / water quality solver telemac3d the 3D hydrodynamic / tracer / water quality solver artemis the phase resolving wave solver tomawac the 3rd generation wave transformation solver sisyphe the sediment transport and geomorphogical solver stbtel a pre-processor for the modules postel3d a post-processor for telemac3d '''), usage=' (--help for help)\n---------\n => ' '%(prog)s module [options] casfile(s)\n---------', epilog=('''\nexamples:\n--------- 1: => runcode.py telemac2d -s t2d.cas ---------''')) parser.add_argument("module", default=None, choices=[ 'telemac2d', 'telemac3d', 'artemis', 'tomawac', 'stbtel', 'postel3d', 'sisyphe', 'partel', 'estel3d', 'mascaret' ]) else: parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, description=('''\n %(prog)s is one of the execution launcher for the TELEMAC system. '''), epilog=('''\nexamples:\n--------- 1: => %(prog)s -s t2d.cas ---------''')) parser.set_defaults(module=module) parser = add_runcode_argument(parser, module=module) # Arguments parser.add_argument("args", metavar='cas file(s)', nargs="+") options = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(CFGS.get_root()) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for one configuration only ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Checking if symlink is available if options.use_link and not check_sym_link(options.use_link): raise TelemacException(\ '\nThe symlink option is only ' 'available on Linux systems. ' 'Remove the option and try again') # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads command line arguments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ code_name = options.module cas_files = options.args # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for only one configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # bypass errors and carries on options.bypass = False if options.split or options.merge or options.run: if options.w_dir == '': raise TelemacException(\ '\nPlease use option -w (--workdirectory)' ' with either of the options ' '--split, --run or --merge\n') # parsing for proper naming CFGS.compute_execution_info() cfg = CFGS.configs[CFGS.cfgname] print('\n\nRunning your CAS file(s) for:\n' + '~' * 72 + '\n') CFGS.light_dump() if options.w_dir != '': print(' +> directory ' + options.w_dir) options.tmpdirectory = False print('\n\n' + '~' * 72 + '\n') # >>> Check wether the config has been compiled for the runcode if options.compileonly: cfg['REBUILD'] = 1 if code_name not in cfg['MODULES']: raise TelemacException(\ '\nThe code requested is not installed ' 'on this system : {}\n'.format(code_name)) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ xcpts = Messages() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Run the Code from the CAS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ for cas_file in cas_files: run_study(cas_file, code_name, options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if xcpts.not_empty(): raise TelemacException(\ '\n\nHummm ... I could not complete ' 'my work.\n{}{}'.format('~'*72, xcpts.except_messages())) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nMy work is done\n\n') sys.exit(0)
def main(): """ @brief Main function of validateTELEMAC. """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~ Handles input arguments options = set_parser() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) version = CFGS.configs[CFGS.cfgname].get('version', 'trunk') # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(CFGS.get_root(), version) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ # Checking if symlink is available if options.use_link and not check_sym_link(options.use_link): raise TelemacException(\ '\nThe symlink option is only available on Linux systems. ' 'Remove the option and try again') # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Forces not to use any Xwindows backend for Jenkins ~~~~~~~~~~ if options.vnv_post: import matplotlib.pyplot as plt plt.switch_backend('agg') # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ xcpts = Messages() # ~~~~ Reporting summary ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if options.notebook: type_valid = 'notebooks' else: type_valid = 'examples' report = Report(options.report_name, type_valid) # ~~~ Running validation cfg = config_corrections(options, CFGS.cfgname) if options.notebook: run_validation_notebooks(options, report, xcpts) else: if options.vnv_mode == 'slurm': run_validation_python_slurm(cfg, options, report, xcpts) else: run_validation_python_mpi(cfg, options, report, xcpts) # Writting report if options.report_name != '': report.write() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if xcpts.not_empty(): print('\n\nHummm ... I could not complete my work.\n' '{}\n{}'.format('~' * 72, xcpts.except_messages())) sys.exit(1) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ else: print('\n\nMy work is done\n\n') sys.exit(0)
def run_validation_python_slurm(cfg, options, report, xcpts): """ Run validation for vnv Python scripts slurm mode @param cfg (Dict) Configuration information @param options (ArgumentParser) List of arguments @param report (Report) Time of actions @param xcpts () Error handler """ # Building list of files to run list_files = get_list_python_files(cfg, options) if list_files == []: print("Nothing to run (check tags and rank)") return n_files = len(list_files) root = CFGS.get_root() if options.cleanup or options.full_cleanup: for ifile, py_file in enumerate(sorted(list_files)): clean_vnv_working_dir(py_file, full=options.full_cleanup) return # Making sure that the file is not there before first run jobid_file = options.id_log if path.exists(jobid_file): remove(jobid_file) # ~> First submission run options.vnv_pre = True options.vnv_run = True options.vnv_post = False options.vnv_check = False options.bypass = True if options.hpc_queue == '': raise TelemacException( "Option --queue is mandatary with --vnv-mode=slurm") print(" ~> Submission part") for ifile, py_file in enumerate(sorted(list_files)): print('\n\nValidation < {}/{} > of {}'\ .format(ifile+1, n_files, py_file.replace(root, '<root>'))) run_python(py_file, options, report, xcpts) # Removing from list files all the ones that crashed in the first # submission run run_list_files = list_files.copy() for error in xcpts.messages: if error['name'] in run_list_files: run_list_files.remove(error['name']) # ~> Waiting for jobs to finish jobs = {} jobs_ini = {} run_times = {} crashed = {'failed': [], 'timeout': []} # File is generated by the first run # In case no run was launched in the previous command if not path.exists(jobid_file): actual_len = 0 else: # Building dictionary of jobs: with open(jobid_file, 'r') as f: for line in f.readlines(): job_id, action_path = line.split(';') if job_id == '': raise TelemacException(\ "Error in the job id file. "\ "Generated by hpc_runcode in systel.cfg:\n{}" .format(jobid_file)) jobs[job_id] = action_path.strip('\n') jobs_ini.update(jobs) # Check job status print(" ~> Waiting for completion") prev_len = 0 actual_len = len(jobs) # Waiting time between each check in second wait_time = 10 start_time = time.time() time.sleep(60) while actual_len != 0: # Only printing remaining jobs if there was some changes if prev_len != actual_len: print("Remaining jobs: ", len(jobs)) t1 = time.time() for job_id in list(jobs.keys()): state = check_job_slurm(job_id) # Job crashed if state == 'failed': crashed['failed'].append(jobs[job_id]) del jobs[job_id] # job timed out elif state == 'timeout': crashed['timeout'].append(jobs[job_id]) del jobs[job_id] # Job is done elif state == 'success': run_time = get_job_time_slurm(job_id) run_times[jobs[job_id]] = run_time del jobs[job_id] # Otherwise job is still running t2 = time.time() # Only wait if the loop was done in less than wait_time if (t2 - t1) < wait_time: time.sleep(wait_time) # Update info on len prev_len = actual_len actual_len = len(jobs) elapsed_time = time.time() - start_time time_str = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) print("Waited {} for jobs to complete".format(time_str)) # Adding run times to the report for py_file, run_time in run_times.items(): # Getting absolute name but same as in jobs tmp = py_file.split(sep) # Mascaret does not have a temporary folder # So not splitting at the same index if 'mascaret' in tmp or 'courlis' in tmp: abs_py_file = sep.join(tmp[:-1]) + ".py" action = tmp[-1] else: abs_py_file = sep.join(tmp[:-2]) + ".py" action = tmp[-2] rank = report.values[abs_py_file]['pre']['rank'] report.add_action(abs_py_file, rank, action, run_time, True) # Building new list of files (without the ones that crashed) new_list_files = [] for py_file in run_list_files: # Extract folder of validation from script name (minus estension) py_folder, _ = path.splitext(py_file) failed_action = crashed['timeout'] + crashed['failed'] # Chekc if that folder is in one of the cases that crashed failed = False for action in failed_action: if py_folder + sep in action: failed = True break # If it is next file if failed: continue new_list_files.append(py_file) # Adding exception for all the run that crashed for crash_type, failed in crashed.items(): if failed != []: for fail in failed: xcpts.add_messages([{ 'name': fail, 'msg': 'The job {}'.format(crash_type) }]) print(" ~> Displaying listing of all runs") # Displaying listings (before merging because merging will remove temporary folder) for ddir in jobs_ini.values(): run_dir = ddir.replace('\n', '') print('\n\nListing for {}:'\ .format(run_dir.replace(path.realpath(root), '<root>'))) # If cmdexec hpc mode listing is in the temporary folder if 'hpc_cmdexec' in cfg: for ffile in listdir(run_dir): if ffile[:4] == 'tmp_' and \ path.isdir(path.join(run_dir, ffile)): run_dir = path.join(run_dir, ffile) break for ffile in listdir(run_dir): if ffile.endswith(".out"): with open(path.join(run_dir, ffile), 'r', encoding='utf-8') as f: print(f.read()) # If we are in hpc_cmdexec configuration (only out_telemac is in the batch # job) # Running on more pass to do the merge step # Second run if 'hpc_cmdexec' in cfg: print(" ~> Merging part") options.vnv_pre = True options.vnv_run = True options.vnv_post = False options.vnv_check = False options.bypass = True options.merge = True options.split = False options.run = False options.compileonly = False # Running only on jobs that finished for ifile, py_file in enumerate(sorted(new_list_files)): print('\n\nValidation < {}/{} > of {}'\ .format(ifile+1, n_files, py_file.replace(root, '<root>'))) run_python(py_file, options, report, xcpts) # Second run options.vnv_pre = True options.vnv_run = False options.vnv_post = True options.vnv_check = True options.bypass = True print(" ~> Check + Post-traitment") # Running only on jobs that finished for ifile, py_file in enumerate(sorted(new_list_files)): print('\n\nValidation < {}/{} > of {}'\ .format(ifile+1, n_files, py_file.replace(root, '<root>'))) run_python(py_file, options, report, xcpts, time_from_report=True)
def main(): """ Main function of compactTELEMAC """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + 72 * '~' + '\n') parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, description=('''\n\ Compact the TELEMAC system files, into various archived:\n 1. archiving sources if necessary 2. archiving examples if necessary 3. archiving binaries if necessary 4. ... '''), usage=' (--help for help)\n---------\n =>' ' %(prog)s [options] \n---------') parser = add_config_argument(parser) parser.add_argument( "-a", "--archive-name", metavar="archive name", dest="archive_name", default='', help="specify the archive name, default is taken as the config name") parser.add_argument( "-m", "--modules", metavar="modules", dest="modules", default='', help="specify the list modules, default is taken from config file") parser.add_argument( "--src", action="store_true", dest="src_only", default=False, help="create a zip containing only the sources i.e. the " "bare minimum to use telemac-mascaret") parser.add_argument( "--examples", action="store_true", dest="examplesOnly", default=False, help="create a zip containing only the sources i.e. the " "bare minimum to use telemac-mascaret") options = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) root_dir = CFGS.get_root() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(root_dir) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for only one common root and zipper ~~~~~~~~~~~~~~~~~~~ CFGS.compute_zip_info() cfg = CFGS.configs[CFGS.cfgname] version = cfg['version'] zip_ext = cfg['ZIPPER'] # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ srcOnlly is independent of config ~~~~~~~~~~~~~~~~~~~~~~~~~~~ if options.src_only: # ~~> create the archive directory if options.archive_name != '': archive_name = options.archive_name else: archive_name = 'otm_' + version + '-src' print('\n\nArchive ' + archive_name + '\n' + '~' * 72 + '\n') tmp_dir = path.join(root_dir, archive_name) if path.exists(tmp_dir): remove_directories(tmp_dir) create_directories(tmp_dir) # ~~> copy the content of the following dirs into the archive directory dirs = ['optionals', 'scripts', 'sources', 'documentation', 'configs'] for pid in dirs: input_path = path.join(root_dir, pid) output_path = input_path.replace(root_dir, tmp_dir) copytree(input_path, output_path, ignore=ignore_patterns('.svn', '*.pyc')) print(' +> ' + input_path) # ~~> copy the following files into the archive directory files = ['NEWS.txt', 'README.txt'] for pid in files: input_path = path.join(root_dir, pid) output_path = input_path.replace(root_dir, tmp_dir) copy_file(input_path, output_path) print(' +> ' + input_path) # ~~> prepare an empty diretory for future builds pid = path.join(root_dir, 'builds') output_path = pid.replace(root_dir, tmp_dir) create_directories(output_path) # ~~> zipping the archive directory print('\n... now packaging ' + archive_name) tel_zip(archive_name, tmp_dir, zip_ext) # ~~> cleaning the archive directory print('\n... now cleaning ') remove_directories(tmp_dir) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ examplesOnly is independent of config ~~~~~~~~~~~~~~~~~~~~~~~ elif options.examplesOnly: # ~~> create the archive directory if options.archive_name != '': archive_name = options.archive_name else: archive_name = 'otm_' + version + '-examples' print('\n\nArchive ' + archive_name + '\n' + '~' * 72 + '\n') tmp_dir = path.join(root_dir, archive_name) if path.exists(tmp_dir): remove_directories(tmp_dir) create_directories(tmp_dir) # ~~> copy the content of the following dir into the archive directory dirs = ['examples'] for pid in dirs: input_path = path.join(root_dir, pid) output_path = input_path.replace(root_dir, tmp_dir) copytree(input_path, output_path, ignore=ignore_patterns('.svn', '*.pyc')) print(' +> ' + input_path) # ~~> zipping the archive directory print('\n... now packaging ' + archive_name) tel_zip(archive_name, tmp_dir, zip_ext) # ~~> cleaning the archive directory print('\n... now cleaning ') remove_directories(tmp_dir) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ else: for cfgname in CFGS.configs: if options.modules != '': cfg['modules'] = \ options.modules.replace(',', ' ')\ .replace(';', ' ').replace('.', ' ') # parsing for proper naming CFGS.compute_compact_info() CFGS.light_dump() # ~~ Scans all source files to build a relation database ~~~~~~~~~~~ if cfg['MODULES'] == {}: raise TelemacException(\ '\nNot able to find any modules within' 'your root directory {}\n'.format(cfg['root'])) # ~~> create the archive directory if options.archive_name != '': archive_name = options.archive_name else: archive_name = 'otm_' + version + '-builds-' + cfgname print('\n\nArchive ' + archive_name + '\n' + '~' * 72 + '\n') tmp_dir = path.join(root_dir, archive_name) if path.exists(tmp_dir): remove_directories(tmp_dir) create_directories(tmp_dir) # ~~> copy the content of the following dir # into the archive directory dirs = ['builds' + sep + cfgname, 'scripts', 'sources', 'configs'] for pid in dirs: input_path = path.join(root_dir, pid) output_path = input_path.replace(root_dir, tmp_dir) copytree(input_path, output_path, ignore=ignore_patterns('.svn', '*.pyc')) print(' +> ' + input_path) # ~~> zipping the archive directory print('\n... now packaging ' + cfgname) tel_zip(cfgname, tmp_dir, cfg['ZIPPER']) # ~~> cleaning the archive directory print('\n... now cleaning ') remove_directories(tmp_dir) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nMy work is done\n\n') sys.exit(0)
def run_mascaret(): """ Main function that runs the mascaret executable in the current folder """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n'+72*'~'+'\n') parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, description=('''\n\ Run the mascaret executable in the current folder, given a CAS file. ''')) parser.add_argument("args", nargs='*') # ~~> Environment parser = add_config_argument(parser) parser.add_argument( "-s", "--sortiefile", action="store_true", dest="sortie_file", default=False, help="specify whether there is a sortie file, default is no") parser.add_argument( "-b", "--bypass", action="store_true", dest="bypass", default=False, help="will bypass execution failures and try to carry on " "(final report at the end)") options = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(CFGS.get_root()) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for one configuration only ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if len(options.args) < 1: print('\nThe name of the CAS file is required\n') parser.print_help() sys.exit(1) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads command line arguments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ cas = options.args[0] # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for only one configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ xcpts = Messages() # still in lower case # parsing for proper naming CFGS.compute_execution_info() cfg = CFGS.configs[CFGS.cfgname] create_mascaret_files(cfg, cas) mascaret_exe = path.join(cfg['root'], 'builds', CFGS.cfgname, 'bin', 'mascaret' + cfg['sfx_exe']) _, code = xcpts.run_cmd(mascaret_exe, options.bypass) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reporting errors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if xcpts.not_empty() or code != 0: print('\n\nHummm ... I could not complete my work.\n'+'~'*72 + xcpts.except_messages()) sys.exit(1) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ else: print('\n\nMy work is done\n\n') sys.exit(0)
def main(): """ Main function of compileTELEMAC """ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + 72 * '~' + '\n') parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, description=('''\n Compile the TELEMAC system:\n 1. rescan the tree dependencies if necessary 2. check which files need re-compilation 3. create object files, libraries, executable, and other binaries depending on your configuration settings Work with all active configurations. ''')) parser = add_config_argument(parser) parser.add_argument( "-m", "--modules", metavar="modules", dest="modules", default='', help="specify the list modules . separated, default is taken from config file") parser.add_argument( "-b", "--bypass", action="store_true", dest="bypass", default=False, help="will bypass execution failures and try to carry on " "(final report at the end)") parser.add_argument( "--rescan", action="store_true", dest="rescan", default=False, help="will redo the scan of sources for an update of " "all the cmdf files") parser.add_argument( "--clean", action="store_true", dest="cleanup", default=False, help="will erase all object, executable libraries from folder " "on the selected configs/modules") parser.add_argument( "-j", type=int, dest="ncsize", default=0, help="set the number of core used for the parallel " "compilation of objects") parser.add_argument( "-v", "--verbose", action="store_true", dest="verbose", default=False, help="If given will print every command") options = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(CFGS.get_root()) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Separately dealing with rescan for all configs ? ~~~~~~~~~~~~ for cfgname in CFGS.configs: # Setting configuration name CFGS.cfgname = cfgname # Getting configuration cfg = CFGS.configs[cfgname] if options.cleanup: CFGS.clean_install(cfgname) print('\n\n' + '\n'.join(banner(cfgname))) print('Scanning the source code for:\n' + '~' * 72 + '\n') compute_config(options.cleanup and options.rescan, options.bypass) CFGS.light_dump() # Only if we ask for a scan if options.rescan: update_cmdf(options.bypass, options.cleanup, options.verbose) # /!\ multiple configurations will now generate multiple rescan # (because of tags and adds, specific to some configurations) compute_config(False, options.bypass) # ~~ Scans all cmdf files found in all modules ~~~~~~~~~~~~~~~~~~~~~ # Specifying what module to compile if options.modules == '': modules = [] else: modules = options.modules.split(".") compile_cmdf(options.ncsize, modules, options.verbose) # Compiling api if asked for cfg_opt = cfg.get('options', []) if 'api' in cfg_opt: compile_api_files(silent=not options.verbose, static='static' in cfg_opt, hermes_only='hermes_only' in cfg_opt) print('\n\nMy work is done\n\n') sys.exit(0)
def scan_xcas(fle): """ @brief : read the xml file to extract the list of input file :param fle: xcas file of mascaret computation :return: list of file needed for computation """ inputfile = [] tree = ET.parse(fle) root = tree.getroot() root2 = root[0] # looking for geometry inputfile.append(root2.find('parametresGeometrieReseau')\ .find('geometrie')\ .find('fichier').text) # looking for laws lois = root2.find('parametresLoisHydrauliques').find('lois') for loi in lois: inputfile.append(loi.find('donnees').find('fichier').text) #looking for initial conditions linits = root2.find('parametresConditionsInitiales').find('ligneEau') if linits.find('LigEauInit').text == 'true': inputfile.append(linits.find('fichLigEau').text) #looking for "casier" if root2.find('parametresCasier') is not None: inputfile.append((root2.find('parametresCasier')\ .find('fichierGeomCasiers').text)) #looking for "paramtresPhysique" if root2.find('parametresTraceur') is not None: root_tracer = root2.find('parametresTraceur') if root_tracer.find( 'parametresConcentrationsInitialesTraceur') is not None: inputfile.append(\ root_tracer.find('parametresConcentrationsInitialesTraceur')\ .find('fichConcInit').text) if root_tracer.find('parametresNumeriquesQualiteEau') is not None: inputfile.append(root_tracer.find('parametresNumeriquesQualiteEau')\ .find('fichParamPhysiqueTracer').text) inputfile.append(root_tracer.find('parametresNumeriquesQualiteEau')\ .find('fichMeteoTracer').text) lois = root_tracer.find('parametresLoisTraceur').find('loisTracer') for loi in lois: inputfile.append(loi.find('fichier').text) #looking for "Courlis" if root2.find('parametresGeneraux').find('optionCourlis') is not None: inputfile.append((root2.find('parametresGeneraux')\ .find('fichierMotCleCourlis').text)) casfile = root2.find('parametresGeneraux')\ .find('fichierMotCleCourlis').text print(casfile) dicofile = path.join(CFGS.get_root(), "sources", "mascaret", "data", "dico_Courlis.txt") cas = TelemacCas(casfile, dicofile) geo_courlis = cas.get('FICHIER DE GEOMETRIE COURLIS') inputfile.append(geo_courlis) return inputfile
def main(): """ Main function of doxygenTELEMAC """ bypass = False # /!\ Temporary bypass for subroutine within programs # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n'+'~'*72+'\n') parser = ArgumentParser(\ formatter_class=RawDescriptionHelpFormatter, description=('''\n Generate the DOXYGEN documentation of the whole TELEMAC system. '''), usage=' (--help for help)\n---------\n => '\ '%(prog)s [options] \n---------') parser = add_config_argument(parser) parser.add_argument(\ "-d", "--doxydir", dest="doxyDir", default='', help="specify the root, default is taken from config file") parser.add_argument(\ "-m", "--modules", dest="modules", default='', help="specify the list modules, default is taken from config file") options = parser.parse_args() # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ update_config(options) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ banners ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ svn_banner(CFGS.get_root()) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for one configuration only ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Works for only one configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ cfg = CFGS.configs[CFGS.cfgname] # still in lower case if options.modules != '': cfg['modules'] = options.modules.replace(',', ' ')\ .replace(';', ' ')\ .replace('.', ' ') if options.doxyDir == '': cfg.update({'doxydocs':path.join(cfg['root'], 'documentation', CFGS.cfgname)}) else: cfg.update({'doxydocs':options.doxyDir}) if not path.isdir(cfg['doxydocs']): create_directories(cfg['doxydocs']) # parsing for proper naming CFGS.compute_doxy_info() print('\n\nScanning the source code for:\n'+'~'*72+'\n') CFGS.light_dump() # ~~ Scans all source files to build a relation database ~~ fic, _, _, _, _, _, _, racine = scan_sources(CFGS.cfgname, cfg, bypass) # ~~ Scann all source files to update Doxygen ~~~~~~~~~~~~~~~~ for mod in fic: print('\nCreating the DOXYGEN headers for ' + mod + '\n'+'~'*72+'\n') for ifile in fic[mod]: # ~~ Read the content of the source file ~~~~~~~~~~~~ ilines = get_file_content(ifile) # ~~ Update its Doxygen content ~~~~~~~~~~~~~~~~~~~~~ olines = create_doxygen(ifile, ilines, mod, racine) # ~~ Make sure the distination exists ~~~~~~~~~~~~~~~ ofile = ifile.replace(cfg['root'], cfg['doxydocs']) create_directories(path.dirname(ofile)) # ~~ Write the content of the source file ~~~~~~~~~~~ put_file_content(ofile, olines) # ~~ Run Doxygen ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\nNow running DOXYGEN within ' + cfg['doxydocs'] + '\n'+'~'*72+'\n') chdir(cfg['doxydocs']) if not path.exists(cfg['cmd_doxygen']): raise Exception('Do not know where to find {}\n ' '... you can correct this through the key ' 'cmd_doxygen in your configuration file' ''.format(cfg['cmd_doxygen'])) if sp.call([cfg['cmd_doxygen']]): raise Exception # ~~ Scan all HTML files and replace template in phases replace_doxygen(path.join(cfg['doxydocs'], 'html')) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nMy work is done\n\n') sys.exit(0)
def main(): """ Main program for the execution of damocles """ # ~~ Reads config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nLoading Options and Configurations\n' + '~' * 72 + '\n') parser = ArgumentParser() parser = add_config_argument(parser) parser.add_argument(\ "-m", "--modules", dest="modules", default='', help="specify the list modules, default is "\ "taken from config file") parser.add_argument(\ "--dump", action="store_true", dest="dump", default=False, help="Will dump a reordered dictionary by rubrique") parser.add_argument(\ "--dump2", action="store_true", dest="dump2", default=False, help="Will dump a reordered dictionary by index") parser.add_argument(\ "--eficas", action="store_true", dest="eficas", default=False, help="Will generate the eficas Catalogue from the dictionary") parser.add_argument(\ "--latex", action="store_true", dest="latex", default=False, help="Will generate the LaTeX file for the reference manual") # ~~~~ Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ args = parser.parse_args() # path to the root # ~~~~ Works for all configurations unless specified ~~~~~~~~~~~~~~~ update_config(args) cfg = CFGS.configs[CFGS.cfgname] # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # Defining which modules to use if args.modules is '': module_list = [ 'artemis', 'postel3d', 'stbtel', 'sisyphe', 'telemac2d', 'telemac3d', 'tomawac', 'waqtel', 'gaia', 'khione' ] else: module_list = args.modules.split(';') # Identify Root value CFGS.compute_vnv_info() root = CFGS.get_root() exe_path = path.join(root, 'builds', CFGS.cfgname,\ 'bin', 'damocles'+\ cfg['SYSTEM']['sfx_exe']) # Looping on all modules for module in module_list: module_path = path.join(root, 'sources', module) if args.dump: input_dict = path.join(module_path, module + ".dico") output_dict = path.join(module_path, module + "2.dico") gen_dump(exe_path, input_dict, output_dict) if args.dump2: input_dict = path.join(module_path, module + ".dico") output_dict = path.join(module_path, module + "2.dico") gen_dump2(exe_path, input_dict, output_dict) if args.eficas: # Creating eficas folder and __init__ if it does not exists eficas_path = path.join(root, 'scripts', 'python3', 'eficas') if not path.exists(eficas_path): mkdir(eficas_path) with open(path.join(eficas_path, '__init__.py'), 'w') as fobj: fobj.write("#! /usr/bin/env python") fobj.write("# -*- coding: utf-8 -*") input_dict = path.join(module_path, module + ".dico") input_dep = path.join(module_path, module + ".dico.dep") fancy_module = module cata_name = path.join(eficas_path, fancy_module + "_cata_auto.py") enum_name = path.join(eficas_path, fancy_module + "_enum_auto.py") ts_path = eficas_path gen_cata(module.upper(), exe_path, input_dict, input_dep, cata_name, enum_name, ts_path + path.sep) if args.latex: input_dict = path.join(module_path, module + ".dico") latex_name = path.join(root, 'documentation', module, 'reference',\ 'latex', 'Corpus.tex') # English only lng = '2' gen_latex(exe_path, input_dict, latex_name, lng) # ~~~~ Compile the valiation documentation print('\n\n' + '~' * 72) # ~~~~ Jenkins' success message ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print('\n\nMy work is done\n\n') sys.exit(0)