def SetParser(run): """Configure the command-line parser, add options name to store to the list, set actions informations. run : AsterRun object which manages the execution """ acts_descr = { 'run': { 'method': RunAster, 'syntax': '[options] profile', 'help': _(u'Execute the execution described by the profile (default action)' ) }, 'quick': { 'method': StartNow, 'syntax': '[options] file1 [file2 [...]] [--surch_pyt=...] [--surch_fort=...]', 'help': _(u'Start quickly an interactive execution using the files given in arguments' ) }, 'test': { 'method': StartTestNow, 'syntax': '[options] testcase [results_directory]', 'help': _(u'Start an interactive execution of a testcase') }, 'make_shared': { 'method': MakeShared, 'syntax': '--output=FILE [src1 [...]] srcN', 'help': _(u'Produce a shared library named FILE by compiling the ' 'source files src1... srcN. Typically used to build a UMAT library.' ), }, } opts_descr = { 'copy_all_results': { 'args': ('--copy_all_results', ), 'kwargs': { 'action': 'store_true', 'default': False, 'dest': 'copy_all_results', 'help': _(u'copy all results in the current directory (for --quick action)' ) } }, 'debugger': { 'args': ('--debugger', ), 'kwargs': { 'action': 'store_true', 'default': False, 'dest': 'debugger', 'help': _(u'run in the debugger (for --quick/--test action)') } }, 'exectool': { 'args': ('--exectool', ), 'kwargs': { 'action': 'store', 'default': False, 'dest': 'exectool', 'help': _(u'run using the specified tool (for --quick/--test action)') } }, 'run_params': { 'args': ('--run_params', ), 'kwargs': { 'action': 'append', 'default': [], 'dest': 'run_params', 'help': _(u'list of parameters added in the export file ' '(for --quick/--test action). ' 'Example: --run_params=actions=make_env will set "P actions make_env"' ' in the export file') } }, } run.SetActions(actions_descr=acts_descr, actions_order=['run', 'quick', 'test', 'make_shared'], group_options=False, options_descr=opts_descr)
dico['name'] = dico['name'].replace('/cygdrive/', '') if dico['name'].find(':/') < 0: dico['name'] = dico['name'].replace('/', ':/', 1) os.makedirs(dico['name']) except OSError, s: # maybe simultaneously created by another process if not os.path.isdir(dico['name']): iret = 4 try: os.chmod(dico['name'], chmod) except OSError, s: self._mess(ufmt(_(u'can not change permissions on %s'), rep), '<A>_ALARM') self.VerbEnd(iret, s, verbose) if iret != 0: self._mess(ufmt(_(u'can not create directory %s'), rep), niverr) return iret def Delete(self, rep, remove_dirs=True, verbose=None): """Delete a file or a directory tree (rm -rf ...). Set 'remove_dirs' to False to be sure to delete 'rep' only if it's a file. """ if verbose == None: verbose = self.verbose iret = 0 dico = self.filename2dict(rep) # preventing to delete first level directories (as /, /home, /usr...) if dico['name'][0] == '/' and len(dico['name'][:-1].split( os.sep)) <= 2: self._mess(ufmt(_(u'deleting this directory seems too dangerous. ' \ '%s has not been removed'), dico['name']), '<A>_ALARM')
def Copy(self, dest, *src, **opts): """Copy files/directories (from 'src' list) to 'dest' Remote names format : [[user[:passwd]@]machine:]name where name is an absolute or a relative path. optional arguments : verbose, follow_output passed to local_shell, niverr : default is set to <F>_COPY_ERROR, but may be '<A>' or less ! protocol : to use a different protocol just for this copy. Note : if src[i] is a directory and dest doesn't exist dest is created and the content of src[i] is copied into dest. This is not a default behavior of rsync which would create `basename src[i]` in dest even if dest doesn't exist. """ iret = 0 kargs = {} # take values from opts if exist kargs['verbose'] = opts.get('verbose', self.verbose) kargs['follow_output'] = opts.get('follow_output', False) kargs['separated_stderr'] = False if opts.get('niverr'): niverr = opts['niverr'] else: niverr = '<F>_COPY_ERROR' ddest = self.filename2dict(dest) if self.IsRemote(dest) \ or (ddest['user'] != '' and ddest['user'] != local_user): fdest = ddest['user'] + '@' + ddest['mach'] + ':' + ddest['name'] else: fdest = ddest['name'] self._dbg('source list : %s' % (src, ), 'destination : %s' % fdest, stack_id=2) if len(src) < 1: self._mess(_(u'no source file to copy'), '<A>_ALARM') for f in src: # here because we can change 'proto' if necessary proto = opts.get('protocol', self.param['remote_copy_protocol']) jret = 0 df = self.filename2dict(f) if self.IsRemote(f): fsrc = df['user'] + '@' + df['mach'] + ':' + df['name'] else: fsrc = df['name'] df['user'] = df['mach'] = '' cmd = '' tail = '.../' + '/'.join(f.split('/')[-2:]) if not opts.has_key('alt_comment'): kargs['alt_comment'] = ufmt(_(u"copying %s..."), tail) else: kargs['alt_comment'] = opts['alt_comment'] if df['mach'] == '' and ddest['mach'] == '': cmd = command['copy'] % {"args": fsrc + ' ' + fdest} else: if proto == 'RSYNC' and df['mach'] != '' and ddest[ 'mach'] != '': proto = 'RCP' self._mess(_(u"copying a remote file to another remote server " \ "isn't allowed through RSYNC, trying with RCP.")) if proto == 'RCP': cmd = 'rcp -r ' + fsrc + ' ' + fdest elif proto == 'SCP': cmd = 'scp -rBCq -o StrictHostKeyChecking=no ' + fsrc + ' ' + fdest elif proto == 'RSYNC': if self.IsDir(f) and not self.Exists(dest): self.MkDir(dest) cmd = 'rsync -rz ' + os.path.join(fsrc, '*') + ' ' + fdest else: cmd = 'rsync -rz ' + fsrc + ' ' + fdest elif proto == 'HTTP': str_user = '' if not df['user'] in ('', 'anonymous'): str_user = df['user'] + '@' # dest must be local if ddest['mach'] == '': cmd = 'wget http://' + str_user + df['mach'] + df[ 'name'] + ' -O ' + fdest else: cmd = '' self._mess(ufmt(_(u'remote destination not allowed through %s' \ ' : %s'), proto, fdest), niverr) if cmd != '': jret, out = self.local_shell(cmd, **kargs) if jret != 0 and niverr != 'SILENT': self._mess(ufmt(_(u'error during copying %s to %s'), f, fdest) \ + os.linesep + ufmt(_(u'message : %s'), out), niverr) else: self._mess(_(u'unexpected error or unknown copy protocol : %s') \ % proto, niverr) iret = max(jret, iret) return iret
def GetMessageInfo(run, *args): """Return info about Code_Aster messages. """ if not run.get('aster_vers'): run.parser.error( _(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option." )) REPREF = run.get_version_path(run['aster_vers']) fconf = run.get('config') if fconf: fconf = osp.abspath(fconf) conf = build_config_of_version(run, run['aster_vers'], fconf) if run['nolocal']: run.Mess( _(u'This operation only works on local source files. "--nolocal" option ignored' )) bibfor = [ conf['SRCFOR'][0], ] if conf['SRCF90'][0] != '': bibfor.append(conf['SRCF90'][0]) args = list(args) named_actions = ( 'check', 'move', ) action = None if len(args) > 0 and args[0] in named_actions: action = args.pop(0) elif len(args) == 0: run.Mess(_(u'You must choose an operation from %s or give a subroutine name or a message ID') \ % repr(named_actions), '<F>_INVALID_ARGUMENT') # surcharge surch_fort = run.get('surch_fort', []) if surch_fort: surch_fort = surch_fort.split(',') surch_pyt = run.get('surch_pyt', []) if surch_pyt: surch_pyt = surch_pyt.split(',') pick_cache = 'messages_dicts.%s.pick' % (run['aster_vers'].replace( os.sep, '_')) msgman = MESSAGE_MANAGER( repref=REPREF, fort=bibfor, pyt=conf['SRCPY'][0], capy=conf['SRCCAPY'][0], cache_dict=osp.join(run['cache_dir'], pick_cache), force=run['force'], verbose=run['verbose'], debug=run['debug'], surch_fort=surch_fort, surch_pyt=surch_pyt, unig=run.get('unigest', None), ) msgman.read_cata() if action == 'check': try: unused, not_found = msgman.check_cata() except CataMessageError, msg: run.Sortie(4) if not run['silent']: print3('Messages inutilises :') print3(' '.join(unused)) print3('Messages inexistants :') print3(' '.join(not_found)) if len(unused) + len(not_found) > 0: run.Sortie(4)
def start(self, options=''): """Go ! """ self.is_starting() self.run.DBG('Profile to run', self.prof.get_content(), all=True) # ----- copy and read .export, build dict for formatting self.build_dict_info(options) if self.run.get('log_usage_version') and self.serv != 'testcase': from asrun.contrib.log_usage import log_usage_version_unfail log_usage_version_unfail(self.run['log_usage_version'], self.prof) jn = self.pid self.name = self.prof['nomjob'][0] if self.run.get(self.mode) not in YES_VALUES: print3( ufmt( _(u"the configuration file (%s) does not allow mode='%s'"), osp.join(confdir, "asrun"), self.mode)) return 4, "" # export file is not necessary in interactive mode if self.mode == 'batch': self.prof.WriteExportTo(self.prof.get_filename()) self.run.DBG('profile written into :', self.prof.get_filename()) # ----- consbtc ? fbtc = osp.join(self.run['flasheur'], 'btc.%s' % jn) if self.prof['consbtc'][ 0] in NO_VALUES and not 'make_env' in self.prof['actions']: fbtc0 = self.prof.Get('D', 'btc')[0]['path'] iret = self.run.Copy(fbtc, fbtc0) self.change_btc(fbtc) else: self.consbtc(fbtc) # ----- soumbtc ? iret = 0 if self.prof['soumbtc'][0] not in NO_VALUES: if self.mode == 'interactif': iret, jobid, self.queue = self.soumbtc_interactif(fbtc) else: iret, jobid, self.queue = self.soumbtc_batch(fbtc) if iret == 0: self.jobid = jobid # copy fbtc into flasheur/ (already removed if run in foreground) if osp.exists(fbtc): jret = self.run.Copy(self.flash('script'), fbtc, niverr='<A>_ALARM') # faut-il recopier le btc vers le client res_fbtc = self.prof.Get('R', 'btc') if len(res_fbtc) > 0: res_fbtc = res_fbtc[0]['path'] self.run.Copy(res_fbtc, fbtc) print3("BTCFILE=%s" % res_fbtc) return iret, ''
class MESSAGE_MANAGER(object): """Classe pour récupérer des informations sur le catalogue de messages. """ def __init__(self, repref, fort=('bibfor', 'bibf90'), pyt='bibpyt', capy='catapy', cache_dict=None, verbose=True, force=False, ignore_comments=True, debug=False, surch_fort=[], surch_pyt=[], unig=None): """Initialisations.""" if type(fort) not in (list, tuple): fort = [ fort, ] if type(surch_fort) not in (list, tuple): surch_fort = [ surch_fort, ] if type(surch_pyt) not in (list, tuple): surch_pyt = [ surch_pyt, ] self.repref = repref self.fort = [osp.join(repref, rep) for rep in fort] self.pyt = osp.join(repref, pyt) self.capy = osp.join(repref, capy) self.surch_fort = [osp.abspath(d) for d in surch_fort] self.surch_pyt = [osp.abspath(d) for d in surch_pyt] self.verbose = verbose self.debug = debug self.force = force self.message_dir = 'Messages' self.cache_dict = cache_dict or osp.join('/tmp/', 'messages_dicts.pick') self.ignore_comments = ignore_comments self.wrk_fort = osp.join(os.getcwdu(), 'F') self.wrk_pyt = osp.join(os.getcwdu(), 'PY') self.unig = unig # init AsterConfig, AsterBuild objects fconf = osp.join(self.repref, 'config.txt') self.conf_obj = AsterConfig(fconf, run=None) self.build_obj = AsterBuild(run=None, conf=self.conf_obj) if unig: # use with_fordepla=False not to mark moved files as removed self.unig = unigest2dict(unig, self.conf_obj, with_fordepla=True) if self.verbose: print3('Repertoires :\n REF=%s\n FORTRAN=%s\n PYTHON=%s\n CAPY=%s' \ % (self.repref, self.fort, self.pyt, self.capy)) self.l_cata = self._get_list_cata() self.l_src, self.l_pyt = self._get_list_src() # self._build_regexp() def read_cata(self): """Read all catalogues""" self._build_dict() self.print_stats() def _filename_cata(self, cata): """Retourne le nom du fichier associé à un catalogue.""" name = osp.join(self.pyt, self.message_dir, cata + '.py') return name def _id2filename(self, msgid): """Retourne modelisa5 à partir de MODELISA5_46""" mat = re.search('([A-Z]+[0-9]*)_([0-9]+)', msgid) if mat == None: raise MessageError(msgid, _(u'invalid message id')) cata, num = mat.groups() return self._filename_cata(cata.lower()), int(num) def _filename2id(self, fcata, num): """Retourne MODELISA5_46 à partir de modelisa5""" return '%s_%d' % (osp.splitext(osp.basename(fcata))[0].upper(), num) def makedirs(self): """Crée les répertoires pour les fichiers modifiés """ for d in (self.wrk_fort, self.wrk_pyt): if not osp.isdir(d): #peter.zhang, for cygwin d = d.replace('\\', '/') d = d.replace('/cygdrive/', '') if d.find(':/') < 0: d = d.replace('/', ':/', 1) os.makedirs(d) def _get_list_cata(self): """Retourne la liste des catalogues de messages. """ re_mess = re.compile('#@.*Messages') s_cata = set(glob(osp.join(self.pyt, self.message_dir, '*.py'))) s_suppr = set() l_surch = [] # ajouter la surcharge et retirer le catalogue d'origine for dsrc in self.surch_pyt: for f in glob(osp.join(dsrc, '*.py')) + glob( osp.join(dsrc, '*', '*.py')): txt = open(f, 'r').read() if re_mess.search(txt): l_surch.append(osp.abspath(f)) fsup = osp.join(self.pyt, self.message_dir, osp.basename(f)) if osp.exists(fsup): s_suppr.add(fsup) s_cata.update(l_surch) if len(l_surch) > 0: print3('%5d catalogues en surcharge : ' % len(l_surch)) print3(os.linesep.join(l_surch)) s_cata = set([osp.abspath(f) for f in s_cata \ if not osp.basename(f) in ['__init__.py', 'context_info.py', 'fermetur.py']]) l_suppr = [] if self.unig and len(self.unig['py']) > 0: l_suppr = [osp.abspath(osp.join(self.repref, f)) \ for f in self.unig['py'] if f.find(self.message_dir) > -1] s_suppr.update(l_suppr) l_suppr = list(s_suppr) l_suppr.sort() if len(l_suppr) > 0: print3('%5d catalogue(s) supprime(s) :' % len(l_suppr)) if self.verbose: print3(os.linesep.join(l_suppr)) l_cata = list(s_cata.difference(s_suppr)) l_cata.sort() return l_cata def check_cata(self): """Vérifie les catalogues. """ if self.verbose: print3('%5d catalogues dans %s + %s' \ % (len(self.l_cata), self.pyt, self.surch_pyt)) all_msg = [] error_occurs = False for f in self.l_cata: try: cata = CATA(f) cata.check() all_msg.extend([self._filename2id(f, i) for i in cata]) if self.verbose: print3(cata) except CataMessageError, msg: error_occurs = True print3(msg) self.l_cata.remove(f) if error_occurs: raise CataMessageError(_(u"global"), _(u"errors occurred!")) all_msg = set(all_msg) # messages jamais appelés used = set(self.d_msg_call.keys()) unused = list(all_msg.difference(used)) unused.sort() union = used.union(all_msg) not_found = list(used.difference(all_msg)) not_found.sort() if self.verbose: print3('%6d messages dans les catalogues' % len(all_msg)) print3('dont %6d messages appeles presents dans les catalogues' % (len(used) - len(not_found))) print3(' et %6d messages inutilises' % len(unused)) print3(' + %6d messages appeles absents des catalogues' % len(not_found)) #print '%6d messages total (union pour verif)' % len(union) return unused, not_found
def __delitem__(self, key): """Supprime le message d'indice 'key'. """ if self[key] == None: raise MessageError(key, _(u'this message does not exist !')) del self.cata_msg[key]
from asrun.common.i18n import _ from asrun.mystring import print3, ufmt, add_to_tail, indent from asrun.thread import Dispatcher from asrun.repart import ResourceManager from asrun.common.utils import now, version2tuple, YES_VALUES from asrun.common.sysutils import local_host, same_hosts, short_hostname from asrun.distrib import DistributionTask from asrun.client import ( ClientConfig, AsterCalcHdlrMulti, SERVER_CONF, MULTIDIR, ) fmt_head = '%s %s %s %s %s %s' % ( _(u"job").center(12), _(u"result").center(18), _(u"cpu").rjust(8), _(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8)) fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f' fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \ % (_("jobs").ljust(7), _("errors").ljust(10)) fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 class DistribMultipleTask(DistributionTask): """Manage several Code_Aster executions. items are couples (username, hostname) attributes (init during instanciation) : IN : run : AsterRun object hostrc : ResourceManager object prof : AsterProfil object
def Multiple(run, prof, runner, numthread='auto'): """Run a multiple execution. """ run.print_timer = True # 1. ----- initializations jn = run['num_job'] # 1.2. rep_trav from profile or from run[...] reptrav = runner.reptrav() run.Mess(_(u'Code_Aster multiple execution'), 'TITLE') runner.set_cpuinfo(1, 1) # ----- how many threads ? try: numthread = int(numthread) except (TypeError, ValueError): numthread = run.GetCpuInfo('numthread') # 1.3. content of the profile serv_list = prof['multiple_server_list'][0] if not serv_list: run.Mess( _(u'List of servers ("multiple_server_list" parameter) not found'), '<F>_ERROR') serv_list = serv_list.split() nbval = len(serv_list) # tell if results have to be transfered on the client server # or let on each host. result_on_client = prof['multiple_result_on_client'][0] in YES_VALUES # this hostrc object is only used to check the hosts availability tit = _(u"Checking hosts") run.timer.Start(tit) client = ClientConfig(run.rcdir) client.init_server_config() avail_servers = client.get_server_list() dhost = {} couples = [] for sname in serv_list: found = False for label in avail_servers: cfg = client.get_server_config(label) if same_hosts(sname, cfg['nom_complet']): found = True client.refresh_server_config([label]) cfg = client.get_server_config(label) if version2tuple(cfg['asrun_vers']) >= (1, 9, 2): couples.append((short_hostname(sname), cfg)) dhost[sname] = {'user': cfg['login']} else: run.Mess(ufmt(_(u"Version 1.9.2 or newer is required to run " \ "multiple executions. It is %s on %s (installed in %s)."), cfg['asrun_vers'], label, cfg['rep_serv']), '<E>_ERROR') break if not found: run.Mess( ufmt(_(u"Host '%s' is not available in %s."), sname, client.rcfile(SERVER_CONF)), "<E>_ERROR") run.DBG("couples :", couples, all=True) hostrc = ResourceManager(dhost) n_avail, n_tot = hostrc.CheckHosts(run, numthread=numthread) run.timer.Stop(tit) run.Mess( _(u'Number of available hosts : %d/%d') % (n_avail, n_tot), "SILENT") if n_avail < 1: run.Mess(_(u"No available host. Run cancelled."), "<F>_ERROR") if n_avail < len(serv_list): run.Mess(_(u"All the hosts are not available. Run cancelled."), "<F>_ERROR") # define a hostrc object to allow a lot of simultaneous executions hostinfo = {local_host: {'mem': 999999, 'cpu': 999999}} hostrc = ResourceManager(hostinfo) #XXX tpsjob : max elapsed time ? try: timeout = prof.get_timeout() * 2. except Exception, reason: run.Mess( _(u"incorrect value for tpsjob : %s") % reason, '<F>_INVALID_PARAMETER')
def TestList(run, *args): """Build a list of testcases. """ if not run.get('aster_vers'): run.parser.error( _(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option." )) REPREF = osp.join(aster_root, run['aster_vers']) fconf = run.get('config') if fconf: fconf = osp.abspath(fconf) conf = build_config_of_version(run, run['aster_vers'], fconf) l_ct = list(args) if len(l_ct) < 1 and not (run['all_test'] or run.get('test_list')): run.parser.error( _(u"'--%s' : you must give testcase names or use one of --all/--test_list option") \ % run.current_action) if run.get('test_list'): iret, l_lue = get_list(run['test_list']) if iret == 0: l_ct.extend(l_lue) else: run.Mess( ufmt(_(u'error during reading file : %s'), run['test_list']), '<F>_ERROR') l_dirs = run.get('astest_dir', ','.join(conf['SRCTEST'])) l_dirs = [ osp.abspath(osp.join(REPREF, p.strip())) for p in l_dirs.split(',') ] run.PrintExitCode = False if run['debug']: run.DBG('astest_dir', l_dirs) run.DBG('filter', run.get('filter')) run.DBG('command', run.get('command')) run.DBG('search', run.get('search')) filtre = LISTE_CT(astest_dir=l_dirs, all=run['all_test'], verbose=not run['silent']) filtre.add_ct(l_ct) res = [] # add filters on parameters if run.get('filter'): for expr in run['filter']: filtre.add_filter('para', expr) if run['debug']: res.append('%% filter on parameter : %s' % expr) # add filters based on command used for expr in run.get('command', []): filtre.add_filter('command', expr) if run['debug']: res.append('%% filter on command : %s' % expr) # add filters based on regular expression for expr in run.get('search', []): filtre.add_filter('regexp', expr) if run['debug']: res.append('%% filter on regexp : %s' % expr) # add user filters if run.get('user_filter'): for f in run['user_filter']: filtre.add_filter('user', f) if run['debug']: res.append('%% user filter, file : %s' % f) res.extend(filtre.build_list()) result = os.linesep.join(res) if run.get('output'): open(run['output'], 'w').write(result) print3( ufmt(_(u'The results have been written into the file : %s'), run['output'])) else: print3(result)
def __init__(self, s_regexp, flags=re.MULTILINE | re.DOTALL): """s_regexp : chaine de l'expression qui doit être vérifiée. """ print3(ufmt(_(u" - regular expression : %s"), repr(s_regexp))) self.regexp = re.compile(s_regexp, flags)
def RunAstout(run, conf, prof, runner, numthread='auto', **kargs): """Run a list of test cases... """ run.print_timer = True # 1. ----- initializations REPREF = prof.get_version_path() # 1.2. rep_trav from profile or from run[...] reptrav = runner.reptrav() run.Mess(_(u'Code_Aster tests execution'), 'TITLE') runner.set_cpuinfo(1, 1) # ----- how many threads ? try: numthread = int(numthread) except (TypeError, ValueError): numthread = run.GetCpuInfo('numthread') # 1.3. content of the profile ltest = osp.join(reptrav, 'tests.list') if not prof.Get('D', typ='list'): run.Mess(_(u'no list of tests found'), '<E>_NO_TEST_LIST') else: for dli in prof.Get('D', typ='list'): if run.IsRemote(dli['path']): tmplist = get_tmpname(run, run['tmp_user'], basename='list') run.ToDelete(tmplist) kret = run.Copy(tmplist, dli['path']) run.FileCat(tmplist, ltest) else: tmplist = run.PathOnly(dli['path']) run.FileCat(tmplist, ltest) if not prof.Get('D', typ='rep_test'): reptest = [] else: reptest = [r['path'] for r in prof.Get('D', typ='rep_test')] if not prof.Get('R', typ='resu_test'): run.Mess(_(u'no result directory found'), '<E>_NO_RESU_DIR') else: resutest = prof.Get('R', typ='resu_test')[0]['path'] if run.IsRemote(resutest): run.Mess(_(u'the result directory must not be on a remote host'), '<F>_ERROR') resutest = run.PathOnly(resutest) #peter.zhang, for cygwin resutest = resutest.replace('\\', '/') resutest = resutest.replace('/cygdrive/', '') if resutest.find(':/') < 0: resutest = resutest.replace('/', ':/', 1) run.MkDir(resutest) run.Delete(osp.join(resutest, 'RESULTAT')) run.Delete(osp.join(resutest, 'NOOK')) flashdir = osp.join(resutest, 'flash') prfl = prof.Get('R', typ='flash') if prfl: if prfl[0]['path'] == "None": flashdir = None else: flashdir = prfl[0]['path'] facmtps = 1. nbmaxnook = 5 cpresok = 'RESNOOK' try: if prof['facmtps'][0] != '': facmtps = float(prof['facmtps'][0]) except ValueError: run.Mess( _(u'incorrect value for %s : %s') % ('facmtps', prof['facmtps'][0])) try: if prof['nbmaxnook'][0] != '': nbmaxnook = int(prof['nbmaxnook'][0]) except ValueError: run.Mess( _(u'incorrect value for %s : %s') % ('nbmaxnook', prof['nbmaxnook'][0])) if prof['cpresok'][0] != '': cpresok = prof['cpresok'][0] run.CheckOK() # get the list of the tests to execute iret, list_tests = get_list(ltest, unique=True) if iret != 0: run.Mess(_(u'error during reading file : %s') % ltest, '<F>_ERROR') nbtest = len(list_tests) # should we run only nook tests ? if run['only_nook'] or prof['only_nook'][0] in YES_VALUES: l_dirs = reptest + [resutest] old_run_result = getDiagnostic(run, l_dirs, list_tests) run.DBG('getDiagnostic result', old_run_result, all=True) nbini = len(list_tests) list_tests = [] for test, dres in old_run_result.items(): if test.startswith('__'): continue if run.GetGrav(dres['diag']) >= run.GetGrav('NOOK'): list_tests.append(test) nbtest = len(list_tests) print3( _(u""" --- %d test-cases to run initially %d test-cases previously failed (or their results have not been found) and will be run again """) % (nbini, nbtest)) random.shuffle(list_tests) # suivi des calculs partagé entre 'numthread' threads tit = _(u"Checking hosts") hostrc = get_hostrc(run, prof) run.timer.Start(tit) n_avail, n_tot = hostrc.CheckHosts(run, numthread=numthread) run.timer.Stop(tit) run.Mess( _(u'Number of available hosts : %d/%d') % (n_avail, n_tot), "SILENT") if n_avail < 1: run.Mess(_(u"No available host. Run cancelled."), "<F>_INVALID_PARAMETER") # timeout before rejected a job = tpsjob try: timeout = prof.get_timeout() except Exception, reason: run.Mess( _(u"incorrect value for tpsjob : %s") % reason, '<F>_INVALID_PARAMETER')
import os import os.path as osp import random import cPickle from asrun.common.i18n import _ from asrun.mystring import print3, ufmt from asrun.thread import Dispatcher from asrun.distrib import DistribTestTask from asrun.repart import get_hostrc from asrun.common_func import get_tmpname from asrun.maintenance import get_aster_version, repr_vers, getDiagnostic from asrun.common.utils import get_list, now, YES_VALUES fmt_head = '%s %s %s %s %s %s' % ( _(u"testcase").center(12), _(u"result").center(18), _(u"cpu").rjust(8), _(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8)) fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f' fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \ % (_("tests").ljust(7), _("errors").ljust(10)) fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 def RunAstout(run, conf, prof, runner, numthread='auto', **kargs): """Run a list of test cases... """ run.print_timer = True # 1. ----- initializations REPREF = prof.get_version_path()
def StartNow(run, *args): """Start quickly a simple execution using files in arguments. """ if not run["silent"]: run.Mess( _(u"This functionnality is still in a beta state of development " u"and may be removed a future release, or may never be improved. " u"Please use --silent option to ignore this message the next time." )) # ----- check argument if not run.get('aster_vers'): run.parser.error( _(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option." )) if len(args) < 1: run.parser.error( _(u"'--%s' requires at least one argument") % run.current_action) # build export lf = [osp.abspath(f) for f in args] prof = build_export_from_files(run, lf, with_results=True) prof = use_options(run, prof) # development files surch_pyt = run.get('surch_pyt', []) if surch_pyt: for obj in surch_pyt.split(','): prof.Set( 'D', { 'path': osp.abspath(obj), 'type': 'py', 'ul': 0, 'isrep': osp.isdir(osp.abspath(obj)), 'compr': False, }) exetmp = None surch_fort = run.get('surch_fort', []) if surch_fort: for obj in surch_fort.split(','): prof.Set( 'D', { 'path': osp.abspath(obj), 'type': 'f', 'ul': 0, 'isrep': osp.isdir(osp.abspath(obj)), 'compr': False, }) exetmp = get_tmpname(run, run['tmp_user'], basename='executable') prof.Set( 'DR', { 'path': exetmp, 'type': 'exec', 'ul': 0, 'isrep': False, 'compr': False, }) if exetmp is not None: prof['actions'] = prof['actions'] + ['make_exec'] # try to grab all results files if run["copy_all_results"]: resudir = os.getcwdu() lcomm = prof.get_type('comm') if len(lcomm) > 0: jobname = osp.splitext(osp.basename(lcomm[0].path))[0] else: jobname = 'unamed' add_all_results(prof, resudir, jobname) run.Mess(_(u"Profile used :"), 'TITLE') run.Mess(os.linesep + prof.get_content(), 'SILENT') # execution RunAster(run, prof)
def __str__(self): return ufmt(_(u"%s: %s, error: %s"), self.__class__.__name__, self.fcata, self.reason)
def _mess_timeout(self, dt, timeout, job, refused): """Emit a message when submission timeout occurs.""" self.run.Mess(ufmt(_(u"no submission for last %.0f s " \ "(timeout %.0f s, equal to 2 * the time requested by the main job), " \ "job '%s' cancelled after %d attempts."), dt, timeout, job, refused))
def move(self, oldid, dest, reuse_hole=True): """Déplace le message 'oldid' dans le catalogue 'dest'. """ if self.verbose: print3('--- moving "%s" into "%s"' % (oldid, dest)) # catalogue objects old_f, old_num = self._id2filename(oldid) new_f = self._filename_cata(dest.lower()) # have these catalogues been already changed ? fmod = osp.join(self.wrk_pyt, osp.basename(old_f)) if osp.isfile(fmod): print3('from %s' % fmod) old_f = fmod fmod = osp.join(self.wrk_pyt, osp.basename(new_f)) if osp.isfile(fmod): print3('from %s' % fmod) new_f = fmod old_cata = CATA(old_f) new_cata = CATA(new_f) if self.verbose: print3(old_cata) print3(new_cata) new_num = new_cata.get_new_id(reuse_hole) if new_num < 0: raise CataMessageError( new_f, _(u'no message id available in this catalog')) newid = self._filename2id(new_f, new_num) # check message existence if old_cata[old_num] == None: raise MessageError(oldid, _(u'unknown message')) new_cata[new_num] = old_cata[old_num] del old_cata[old_num] # write modified catalogues self.makedirs() fout = osp.join(self.wrk_pyt, osp.basename(old_f)) content = old_cata.get_content() open(fout, 'w').write(content) fout = osp.join(self.wrk_pyt, osp.basename(new_f)) content = new_cata.get_content() open(fout, 'w').write(content) print3('Nouveau fichier : %s' % fout) # modify source using 'oldid' message l_src = self.d_msg_call.get(oldid, []) for f in l_src: ext = osp.splitext(f)[-1] rdest = self.wrk_fort if ext == '.py': rdest = self.wrk_pyt # already changed ? fmod = osp.join(rdest, osp.basename(f)) if osp.isfile(fmod): print3('from %s' % fmod) f = fmod txt = open(osp.join(self.repref, f), 'r').read() new = re.sub('%s([\'\"]+)' % oldid, newid + r'\1', txt) fout = osp.join(rdest, osp.basename(f)) open(fout, 'w').write(new) print3('Nouveau fichier : %s' % fout)
def _mess_running_timeout(self, dt, timeout, job): """Emit a message when running timeout occured.""" self.run.Mess(ufmt(_(u"The job '%s' has been submitted since %.0f s and is "\ "not ended (timeout %.0f s, equal to 4 * the time asked for the main job). " \ "You can kill it and get other results or wait again..."), job, dt, timeout))
def __setitem__(self, key, msg): """Ajoute le message 'msg' à l'indice 'key'. """ if self[key] != None: raise MessageError(key, _(u'message already exists !')) self.cata_msg[key] = msg
def start(): # ----- divers os.stat_float_times(True) # ----- backward compatibility from asrun.backward_compatibility import change_argv sys.argv = change_argv(sys.argv) # ----- initialisation run = AsterRun() magic.run = run # ----- retrieve options and arguments opts, args = run.ParseArgs() # init magic magic.set_stdout(run['stdout']) magic.set_stderr(run['stderr']) magic.init_logger(filename=run['log_progress'], debug=run['debug']) run.current_action = opts.action if run.current_action == None: # if symbolic link "action" -> "as_run --action" alias = os.path.basename(sys.argv[0]) if alias in run.actions_info.keys(): run.current_action = alias else: # default to 'run' run.current_action = 'run' #run.parser.error(_(u'you must specify an action')) # ----- get system commands run.DBG("Command line run on '%s'" % local_full_host, "using python executable '%s' :" % sys.executable, sys.argv) run.system = AsterSystem(run) run.PostConf() # ----- debug information if run['debug']: run.PrintConfig() print3(_(u'Arguments :'), repr(args)) print3() # ----- start 'current_action' try: act = run.current_action if run.options['proxy'] is True: act = 'call_proxy' meth = run.actions_info[act]['method'] except KeyError: run.Mess( _(u'dictionnary bad defined :') + ' actions_info', '<F>_PROGRAM_ERROR') else: # trap <Control+C> try: meth(run, *args) except KeyboardInterrupt: run.Mess( _(u"'--%s' stopped by user") % run.current_action, '<F>_INTERRUPT') run.Sortie(0)
def __repr__(self): """Résumé du catalogue. """ return ufmt(_(u'%3d messages (last=%3d, next=%3d) in %s'), self.get_nb_msg(), self.get_last_id(), self.get_new_id(), self.fcata)
try: c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug']) except Exception, msg: run.Mess(msg, '<F>_DB_ERROR') typ = ReadDB(TYPE, c) d_typ = { 'AL' : typ['anomalie'], 'EL' : typ['evolution'], 'AOM' : typ['aide utilisation'], 'AO' : typ['anomalie'], 'EO' : typ['evolution'], 'ED' : typ['evolution'], } if not d['TYPFIC'] in d_typ.keys(): run.Mess(_(u'Unknown issue type : %s') % d['TYPFIC'], '<F>_PARSE_ERROR') # 4. create new issue # 4.1. get fields from db login = run.system.getuser_host()[0] loginrex = login try: res = c.exe("""SELECT id, _username FROM _user WHERE _loginaster='%s' AND __retired__=0;""" % login) loginrex = res[0][1] if len(res) > 1: run.Mess(_(u"More than one user has '%s' as login in REX database, " \ "'%s' is taken") % (login, loginrex), '<A>_MULTIPLE_USER') except (IndexError, db.MySQLError), msg: run.Mess(str(msg)) run.Mess(_(u'User %s unknown in REX database') % login,
def soumbtc_interactif(self, fbtc): """Run btc in interactive mode. """ self.jobid = self.pid # commandes cmd_batch = '%(fbtc)s 1> %(output)s 2> %(error)s' cmd_interactif = '%(fbtc)s 2> %(error)s | tee %(output)s' node = self.dict_info['node'] dico = { 'fbtc': fbtc, 'output': self.flash('output'), 'error': self.flash('error'), } # follow output or not xterm = '' if self.prof['follow_output'][0] in YES_VALUES: xterm = self.run['terminal'] if self.prof['depart'][0] != '' or xterm == '': cmd = cmd_batch % dico else: cmd = cmd_interactif % dico # delayed start if self.prof['depart'][0] != '': cmd = "echo '%s' | at %s" % (cmd, self.prof['depart'][0]) elif xterm != '': if re.search('@E', xterm) == None: xterm = xterm + ' -e @E' cmd = xterm.replace('@E', '%s "%s"' % (shell_cmd, cmd)) # run on another node distant = not is_localhost2(node) if not distant: # check xterm command if xterm != '': term = xterm.split()[0] if not os.access(term, os.X_OK): print3(_(u"Not an executable : %s") % term) return 7, '', 'unknown' else: # check node connection iret, output = self.run.Shell('echo hello', mach=node) if output.find('hello') < 0: print3(output) print3( _(u"Connection failure to %s (from %s)") % (node, self.prof['serveur'][0])) return 6, '', 'unknown' # background is not possible if display forwarding is required # (the connection must stay alive) need_display = xterm != '' background = (not need_display) \ or same_hosts2(self.prof['mclient'][0], node, self.prof['uclient'][0], self.prof['username'][0]) kret, output = self.run.Shell(cmd, mach=node, bg=background, display_forwarding=need_display) return 0, self.jobid, 'interactif'
def mark_as_closed(run, l_nf, cnx, tagv, expl): """Mark a list of issues as closed l_nf: list of issues numbers cnx: connection object to database tagv: tag inserted in issues expl: True if it concerns a stable version (exploitation)""" etat = _read_table(run, cnx, STATUS) prod = _read_table(run, cnx, PRODUIT) typv = 'dev' autr = 'expl' if expl: typv = 'expl' autr = 'dev' d_champ = { 'a_corrige' : '_corrV' + typv, 'v_correct' : '_verCorrV' + typv, 'a_tester' : '_corrV' + autr, 'v_tester' : '_verCorrV' + autr, } req_status = 'valide_EDA' for numf in l_nf: # 4.1. read the issue try: issue = ISSUE(numf, cnx) except ReadDBError: run.Mess(_(u'Unable to read issue %s') % numf, '<E>_UNKNOWN_ISSUE') continue # 4.2. check issue values status = issue['_status'].GetPrimValue() if status != req_status: run.Mess(_(u"Status of issue %s is not '%s' (%s)") \ % (numf, req_status, status), '<A>_UNEXPECTED_VALUE') issue['_status'] = etat[req_status] if issue['_produit'].GetPrimValue() == 'Code_Aster': if not issue[d_champ['a_corrige']] \ and issue['_type'].GetPrimValue() != 'aide utilisation': issue[d_champ['a_corrige']] = YES run.Mess(_(u"issue %s should not been solved in version %s") \ % (numf, typv), '<A>_UNEXPECTED_VALUE') else: issue[d_champ['a_corrige']] = NO issue[d_champ['a_tester']] = NO # 4.3. fill issue fields issue[d_champ['v_correct']] = tagv # 4.4. close issue ? if not issue[d_champ['a_tester']] or not issue[d_champ['v_tester']] in ('', None): new_status = etat['attente_doc'] if not issue['_impactDoc']: new_status = etat['ferme'] issue['_status'] = new_status run.Mess(_(u'issue %s is closed') % numf) else: run.Mess(_(u'issue %s must be solved in V%s too') % (numf, autr)) # 4.5. write issue in database try: if not run['debug']: issue.write(force=True) else: issue.repr() except WriteDBError, msg: run.Mess(_(u'error occurs during writing issue'), '<F>_DB_ERROR')
""" import os import os.path as osp from math import log10 from asrun.common.i18n import _ from asrun.mystring import print3, ufmt, cleanCR from asrun.thread import Dispatcher from asrun.distrib import DistribParametricTask from asrun.common_func import get_tmpname from asrun.repart import get_hostrc from asrun.common.utils import now fmt_head = '%s %s %s %s %s %s' % ( _(u"job").center(12), _(u"result").center(18), _(u"cpu").rjust(8), _(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8)) fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f' fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \ % (_("jobs").ljust(7), _("errors").ljust(10)) fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 def Parametric(run, prof, runner, numthread='auto', **kargs): """Run a parametric study. """ run.print_timer = True # 1. ----- initializations jn = run['num_job']
def SetParser(run): """Configure the command-line parser, add options name to store to the list, set actions informations. run : AsterRun object which manages the execution """ acts_descr = { 'create_issue' : { 'method' : Creer, 'syntax' : 'issue_file export_file', 'help' : _(u'Insert a new entry in the issue tracking system and ' 'copy attached files if an export file is provided') }, 'extract_histor' : { 'method' : Histor, 'syntax' : '[--status=STAT] [--format=FORM] [--all_msg] input_file histor', 'help' : _(u'Extract the content of issues listed in `input_file` to `histor`') }, 'close_issue' : { 'method' : Solder, 'syntax' : '--vers=VERS histor', 'help' : _(u'Fill "corrVdev" or "corrVexpl" field (depends on VERS) in issues found in `histor` and eventually close them') }, } opts_descr = { 'status' : { 'args' : ('--status', ), 'kwargs' : { 'action' : 'store', 'default' : 'all', 'metavar' : 'STAT', 'choices' : ('all', 'resolu', 'valide_EDA', 'attente_doc', 'ferme'), 'dest' : 'status', 'help' : _(u'raise an error if issues are not in this status') } }, 'format' : { 'args' : ('--format', ), 'kwargs' : { 'action' : 'store', 'default' : 'text', 'metavar' : 'FORM', 'choices' : ('text', 'html', 'fsq'), 'dest' : 'format', 'help' : _(u'format of generated histor file (text or html)') } }, 'all_msg' : { 'args' : ('--all_msg', ), 'kwargs' : { 'action' : 'store_true', 'default' : False, 'dest' : 'all_msg', 'help' : _(u'retrieve all the messages of issues') } }, } title = _(u'Options for issue tracker interface') run.SetActions( actions_descr=acts_descr, actions_order=['create_issue', 'close_issue', 'extract_histor'], group_options=True, group_title=title, actions_group_title=False, options_descr=opts_descr, )
def local_shell(self, cmd, bg=False, verbose=False, follow_output=False, alt_comment=None, interact=False, separated_stderr=False, stack_id=3, **ignore_args): """Execute a command shell cmd : command bg : put command in background if True verbose : print status messages during execution if True follow_output : follow interactively output of command alt_comment : print this "alternative comment" instead of "cmd" Return : iret : exit code if bg = False, 0 if bg = True output : output lines (as string) """ if not alt_comment: alt_comment = cmd if len(cmd) > self.MaxCmdLen: self._mess((_(u'length of command shell greater '\ 'than %d characters.') % self.MaxCmdLen), '<A>_ALARM') self._dbg('cmd :', cmd, 'background : %s' % bg, 'follow_output : %s' % follow_output, all=True, stack_id=stack_id) self.VerbStart(alt_comment, verbose=verbose) if follow_output and verbose: print3(os.linesep + _(u'Command output :')) var_id = "EXIT_COMMAND_%s" % get_command_id() fout_name = get_tmpname_base(self._tmpdir, 'local_shell_output', user=local_user, node=local_host, pid="auto") ferr_name = get_tmpname_base(self._tmpdir, 'local_shell_error', user=local_user, node=local_host, pid="auto") new_cmd = get_command_line(cmd, bg, follow_output, separated_stderr, fout_name, ferr_name, var_id) # execution iret = os.system(convert(new_cmd)) output, error = "", "" try: output = to_unicode(open(fout_name, "r").read()) os.remove(fout_name) except: pass try: error = to_unicode(open(ferr_name, "r").read()) os.remove(ferr_name) except: pass if follow_output: # repeat header message self.VerbStart(alt_comment, verbose=verbose) mat = re.search('EXIT_CODE=([0-9]+)', output) if mat: iret = int(mat.group(1)) elif follow_output: # os.system returns exit code of tee mat = re.search("%s=([0-9]+)" % var_id, output) if mat: iret = int(mat.group(1)) self.VerbEnd(iret, output, verbose=verbose) if iret != 0: print('ERROR : iret = %s' % iret, '+++ STANDARD OUTPUT:', output, '+++ STANDARD ERROR:') if bg: iret = 0 if not separated_stderr: result = iret, output else: result = iret, output, error return result
try: unused, not_found = msgman.check_cata() except CataMessageError, msg: run.Sortie(4) if not run['silent']: print3('Messages inutilises :') print3(' '.join(unused)) print3('Messages inexistants :') print3(' '.join(not_found)) if len(unused) + len(not_found) > 0: run.Sortie(4) elif action == 'move': if len(args) != 2: run.parser.error( _(u"'--%s %s' requires 2 arguments") % (run.current_action, action)) msgman.move(*args) else: print3() fmt = '%12s : %s' for obj in args: l_msg = list(set(msgman.which_msg(obj))) if len(l_msg) > 0: l_msg.sort() print3(fmt % (obj, l_msg)) l_sub = list(set(msgman.who_use(obj))) if len(l_sub) > 0: l_sub.sort() print3(fmt % (obj, l_sub))
def _gzip_gunzip(self, mode, src, **opts): """Called by Gzip (mode=gzip) and Gunzip (mode=gunzip) methods. Return status and filename of src after de/compression. Only local files/directories supported ! """ para = { 'gzip': { 'cmd': 'gzip ', 'niverr': '<F>_COMPRES_ERROR', 'msg0': _(u'no file/directory to compress'), 'comment': _(u'compressing %s'), 'msgerr': _(u'error during compressing %s'), }, 'gunzip': { # sometimes gunzip doesn't exist, gzip seems safer 'cmd': 'gzip -d ', 'niverr': '<F>_DECOMPRESSION', 'msg0': _(u'no file/directory to decompress'), 'comment': _(u'decompressing %s'), 'msgerr': _(u'error during decompressing %s'), }, } if not mode in para.keys(): self._mess(_(u'unknown mode : %s') % mode, '<F>_PROGRAM_ERROR') iret = 0 if opts.has_key('cmd'): cmd = opts['cmd'] else: cmd = para[mode]['cmd'] if opts.has_key('verbose'): verbose = opts['verbose'] else: verbose = self.verbose if opts.has_key('niverr'): niverr = opts['niverr'] else: niverr = para[mode]['niverr'] if len(src) < 1: self._mess(para[mode]['msg0'], '<A>_ALARM') return iret if not os.path.isdir(src): if mode == 'gzip': name = src + '.gz' else: name = re.sub('\.gz$', '', src) lf = [src] else: name = src lf = glob.glob(os.path.join(src, '*')) for f in lf: jret = 0 comment = para[mode]['comment'] % f if os.path.isdir(f): self.VerbStart(comment, verbose=verbose) self.VerbIgnore(verbose=verbose) else: jret = self.local_shell(cmd + f, verbose=verbose, alt_comment=comment)[0] if jret != 0: self._mess(para[mode]['msgerr'] % f, niverr) iret = max(iret, jret) return iret, name
def RunAster(run, *args): """Allow to run Code_Aster with or without compiling additional source files, check a development, run a list of test cases... """ run.print_timer = True prev = os.getcwdu() # ----- check argument if len(args) != 1: run.parser.error( _(u"'--%s' requires one argument") % run.current_action) # 1. ----- initializations jn = run['num_job'] fprof = get_tmpname(run, run['tmp_user'], basename='profil_astk') run.ToDelete(fprof) # 1.1. ----- check argument type # 1.1.1. ----- use profile from args if isinstance(args[0], AsterProfil): prof = args[0].copy() prof.WriteExportTo(fprof) forig = fprof else: # 1.1.2. ----- read profile from args forig = args[0] kret = run.Copy(fprof, forig, niverr='<F>_PROFILE_COPY') prof = AsterProfil(fprof, run) if not run.IsRemote(forig): export_fname = run.PathOnly(get_absolute_path(forig)) prof.absolutize_filename(export_fname) if not prof['mode'][0] in ('batch', 'interactif'): run.Mess(_(u"Unknown mode (%s), use 'interactif' instead") % \ repr(prof['mode'][0]), 'UNEXPECTED_VALUE') prof['mode'] = ['interactif'] run.DBG("Input export : %s" % fprof, prof) # 1.2. get AsterConfig and AsterBuild objects REPREF = prof.get_version_path() conf = build_config_from_export(run, prof) build = AsterBuild(run, conf) DbgPara = { 'debug': { 'exe': conf['BIN_DBG'][0], 'suff': conf['BINOBJ_DBG'][0], 'libast': conf['BINLIB_DBG'][0], 'libfer': conf['BINLIBF_DBG'][0] }, 'nodebug': { 'exe': conf['BIN_NODBG'][0], 'suff': conf['BINOBJ_NODBG'][0], 'libast': conf['BINLIB_NODBG'][0], 'libfer': conf['BINLIBF_NODBG'][0] }, } # 1.3. set environment depending on version for f in conf.get_with_absolute_path('ENV_SH'): run.AddToEnv(f) # 1.4. set runner parameters klass = Runner # allow to customize of the execution objects if run.get('schema_execute'): schem = get_plugin(run['schema_execute']) run.DBG("calling plugin : %s" % run['schema_execute']) klass = schem(prof) runner = klass(conf.get_defines()) iret = runner.set_cpuinfo(prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0]) if iret == 1: run.Mess( ufmt( _(u"%s is not a MPI version of Code_Aster. " "The number of nodes/processors must be equal to 1."), REPREF), "<F>_INVALID_PARAMETER") elif iret != 0: run.Mess(_(u"incorrect value for mpi_nbnoeud (%s) or mpi_nbcpu (%s)") \ % (prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0]), '<F>_INVALID_PARAMETER') # 1.5. rep_trav from profile or from run[...] reptrav = runner.set_rep_trav(prof['rep_trav'][0], prof['mode'][0]) # write reptrav in the export prof['rep_trav'] = reptrav prof.WriteExportTo(prof.get_filename()) #XXX overrides the original export if forig != prof.get_filename(): run.Copy(forig, prof.get_filename(), niverr='<A>_ALARM') # add reptrav to LD_LIBRARY_PATH (to find dynamic libs provided by user) old = os.environ.get("LD_LIBRARY_PATH", "") os.environ["LD_LIBRARY_PATH"] = (reptrav + os.pathsep + old).strip( os.pathsep) # do not reinitialize rep_trav if if prof['prep_env'][0] not in NO_VALUES: run.MkDir(reptrav, chmod=0700) if prof['detr_rep_trav'][0] not in NO_VALUES: run.ToDelete(reptrav) # 1.6. copy profile in rep_trav kret = run.Copy(osp.join(reptrav, jn + '.export'), fprof) # ... and config file as ./config.txt conf.WriteConfigTo(osp.join(reptrav, 'config.txt')) # 1.7. debug/nodebug dbg = prof['debug'][0] if dbg == '': dbg = 'nodebug' # 1.8. default values exetmp = osp.join(REPREF, DbgPara[dbg]['exe']) cmdetmp = osp.join(REPREF, conf['BINCMDE'][0]) eletmp = osp.join(REPREF, conf['BINELE'][0]) # 2. ----- read profile values # it's valid because exec, cmde and ele must appear only once # these values will be overidden if they are available in reptrav # after an occurence of 'make_...' if prof.Get('DR', 'exec'): exetmp = prof.Get('DR', 'exec')[0]['path'] if prof.Get('DR', 'cmde'): cmdetmp = prof.Get('DR', 'cmde')[0]['path'] if prof.Get('DR', 'ele'): eletmp = prof.Get('DR', 'ele')[0]['path'] # order of actions : list_actions = [ 'make_exec', 'make_cmde', 'make_ele', 'make_etude', 'make_dbg', 'make_env', 'astout', 'distribution', 'multiple', 'exec_crs', 'exec_crp' ] # 3. ==> Let's go ! # 3.0. check if we know what to do for act in prof['actions']: if act == '': run.Mess(_(u'nothing to do'), 'OK') elif not act in list_actions: run.Mess(_(u'unknown action : %s') % act, '<A>_ALARM') # check if the version allows developments if conf['DEVEL'][0] in NO_VALUES and \ ( 'make_exec' in prof['actions'] or \ 'make_cmde' in prof['actions'] or \ 'make_ele' in prof['actions'] ): run.Mess( _(u'The configuration of this version does not allow ' 'user developments.'), '<F>_ERROR') # # 3.1. ----- make_exec # iret = 0 if 'make_exec' in prof['actions']: run.DBG(u'Start make_exec action') exetmp = osp.join(reptrav, 'aster.exe') tit = _(u'Compilation of source files') run.Mess(tit, 'TITLE') run.timer.Start(tit) repact = osp.join(reptrav, 'make_exec') repobj = osp.join(repact, 'repobj') run.MkDir(repact) lf = [] for typ in ('c', 'f', 'f90'): for rep in [l['path'] for l in prof.Get('D', typ=typ)]: jret, lbi = build.Compil(typ.upper(), rep, repobj, dbg, rep_trav=repact, error_if_empty=True, numthread='auto') iret = max(iret, jret) lf.extend(lbi) # liste des fichiers surchargés vers = get_aster_version(REPREF) vers = '.'.join(vers[:3]) fsurch = osp.join(repact, 'surchg.f') listsurcharge(vers, fsurch, lf) jret, lbi = build.Compil('F', fsurch, repobj, dbg, repact) run.timer.Stop(tit) run.CheckOK() tit = _(u'Build executable') run.Mess(tit, 'TITLE') run.timer.Start(tit) libaster = osp.join(REPREF, DbgPara[dbg]['libast']) libferm = osp.join(REPREF, DbgPara[dbg]['libfer']) # for backward compatibility if run.IsDir(libaster): libaster = osp.join(libaster, 'lib_aster.lib') if run.IsDir(libferm): libferm = osp.join(libferm, 'ferm.lib') lobj = glob(osp.join(repobj, '*.o')) # build an archive if there are more than NNN object files if len(lobj) > 500: run.timer.Stop(tit) tit2 = _(u'Add object files to library') run.timer.Start(tit2) libtmp = osp.join(repobj, 'libsurch.a') run.Copy(libtmp, libaster) kret = build.Archive(repobj, libtmp, force=True) lobj = [] libaster = libtmp run.timer.Stop(tit2) run.timer.Start(tit) kret = build.Link(exetmp, lobj, libaster, libferm, repact) run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit, num=999) if prof.Get('R', typ='exec'): exe = prof.Get('R', typ='exec')[0] run.Delete(exe['path'], remove_dirs=False) iret = run.MkDir(osp.dirname(exe['path'])) iret = run.Copy(exe['path'], exetmp) exedata = prof.Get('D', typ='exec') if exedata and exedata[0]['path'] != exe['path']: exetmp = exedata[0]['path'] run.timer.Stop(tit) run.Mess(_(u'Code_Aster executable successfully created'), 'OK') # # 3.2. ----- make_cmde # if 'make_cmde' in prof['actions']: run.DBG(u'Start make_cmde action') tit = _(u"Compilation of commands catalogue") cmdetmp = osp.join(reptrav, 'cata_commande') run.timer.Start(tit) repact = osp.join(reptrav, 'make_cmde') run.MkDir(repact) kargs = { 'exe': exetmp, 'cmde': cmdetmp, } kargs['capy'] = [l['path'] for l in prof.Get('D', typ='capy')] lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] jret = build.CompilCapy(REPREF, repact, **kargs) #i18n=True, run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit) if prof.Get('R', typ='cmde'): cmde = prof.Get('R', typ='cmde')[0] iret = run.MkDir(cmde['path']) iret = run.Copy(cmde['path'], osp.join(cmdetmp, 'cata*.py*')) run.timer.Stop(tit) # # 3.3. ----- make_ele # if 'make_ele' in prof['actions']: run.DBG(u'Start make_ele action') tit = _(u"Compilation of elements") eletmp = osp.join(reptrav, 'elem.1') run.timer.Start(tit) repact = osp.join(reptrav, 'make_ele') run.MkDir(repact) kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, } kargs['cata'] = [l['path'] for l in prof.Get('D', typ='cata')] lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] jret = build.CompilEle(REPREF, repact, **kargs) run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit) if prof.Get('R', typ='ele'): ele = prof.Get('R', typ='ele')[0] iret = run.MkDir(osp.dirname(ele['path'])) iret = run.Copy(ele['path'], eletmp) run.timer.Stop(tit) # # 3.4. ----- make_env / make_etude / make_dbg # if 'make_env' in prof['actions'] or 'make_etude' in prof['actions'] or \ 'make_dbg' in prof['actions']: run.DBG(u'Start make_etude/make_env/make_dbg action') os.chdir(reptrav) run.Mess(_(u'Code_Aster execution'), 'TITLE') # 3.4.1. prepare reptrav to run Code_Aster (proc# = 0) only_env = 'make_env' in prof['actions'] kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'lang': prof['lang'][0], 'only_env': only_env, } lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] tit = _(u'Preparation of environment') run.timer.Start(tit) run.Mess(ufmt(_(u'prepare environment in %s'), reptrav)) if prof['prep_env'][0] != 'no': build.PrepEnv(REPREF, reptrav, dbg=dbg, **kargs) else: run.Mess(_(u'... skipped (%s = no) !') % 'prep_env', 'SILENT') run.timer.Stop(tit) # 3.4.2. copy datas (raise <E> errors if failed) tit = _(u'Copying datas') run.Mess(tit, 'TITLE') run.timer.Start(tit) if prof['copy_data'][0] not in NO_VALUES: copyfiles(run, 'DATA', prof) else: run.Mess(_(u'... skipped (%s = no) !') % 'copy_data', 'SILENT') print3(os.getcwdu()) run.timer.Stop(tit) # 3.4.3. execution diag, tcpu, tsys, ttot, copybase = execute(reptrav, multiple=False, with_dbg='make_dbg' in prof['actions'], only_env=only_env, runner=runner, run=run, conf=conf, prof=prof, build=build, exe=exetmp) if not 'make_env' in prof['actions']: # 3.4.4. copy results tit = _(u'Copying results') run.Mess(tit, 'TITLE') run.timer.Start(tit) if prof['copy_result'][0] not in NO_VALUES: emit_alarm = prof['copy_result_alarm'][0] not in NO_VALUES copyfiles(run, 'RESU', prof, copybase, emit_alarm) else: run.Mess( _(u'... skipped (%s = no) !') % 'copy_result', 'SILENT') run.timer.Stop(tit) run.Mess(_(u'Code_Aster run ended'), diag) # 3.4.5. add .resu/.erre to output for testcases ctest = prof['parent'][0] == "astout" if ctest: run.Mess(_(u'Content of RESU file'), 'TITLE') run.FileCat('fort.8', magic.get_stdout()) run.Mess(_(u'Content of ERROR file'), 'TITLE') run.FileCat('fort.9', magic.get_stdout()) # 3.4.6. notify the user if prof['notify'][0]: content = _( '[Code_Aster] job %(job)s on %(server)s ended: %(diag)s') content = content % { 'job': prof.get_jobname(), 'diag': diag, 'server': prof['serveur'][0], } dest = ','.join(prof['notify']) run.SendMail(dest=dest, text=content, subject=content.splitlines()[0]) run.Mess(_(u'Email notification sent to %s') % dest) run.CheckOK() os.chdir(prev) # 3.5. ----- astout if 'astout' in prof['actions']: run.DBG(u'Start astout action') kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'numthread': prof['numthread'][0], } os.chdir(reptrav) RunAstout(run, conf, prof, runner=runner, **kargs) os.chdir(prev) # 3.6. ----- distribution if 'distribution' in prof['actions']: run.DBG(u'Start distribution action') kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'numthread': prof['numthread'][0], } Parametric(run, prof, runner=runner, **kargs) # 3.7. ----- multiple if 'multiple' in prof['actions']: run.DBG(u'Start multiple action') Multiple(run, prof, runner=runner, numthread=prof['numthread'][0]) # 4. ----- clean up if 'make_env' in prof['actions'] and prof['detr_rep_trav'][ 0] not in NO_VALUES: run.DoNotDelete(reptrav)