def Insert(run, *args): """Insert an execution to the database. """ if len(args) != 1: run.parser.error(_(u"'--%s' takes exactly %d arguments (%d given)") % \ (run.current_action, 1, len(args))) iret = 0 if not run.config.get('astketud'): run.Mess(_(u"'astketud' is not defined in 'agla' configuration file."), '<F>_AGLA_ERROR') # 1. copy export file jn = run['num_job'] fprof = get_tmpname(run, run['tmp_user'], basename='etude_prof') kret = run.Copy(fprof, args[0], niverr='<F>_PROFILE_COPY') # 2. insert study in database cmd = '%(astketud)s %(profile)s' % { 'astketud': run['astketud'], 'profile': fprof, } iret, output = run.Shell(cmd) if iret != 0: run.Mess(output, '<F>_DB_ERROR') # astketud always returns exit=0 ! else: print3(output)
def get_hostrc(run, prof): """Return hostrc object from profile or config. """ mode = prof['mode'][0] hostrc = None if prof.Get('D', 'hostfile'): fhost = prof.Get('D', 'hostfile')[0]['path'] if run.IsRemote(fhost) or True: tmphost = get_tmpname(run, run['tmp_user'], basename='hostfile') run.ToDelete(tmphost) iret = run.Copy(tmphost, fhost) fhost = tmphost else: fhost = run.PathOnly(fhost) elif run.get('%s_distrib_hostfile' % mode): fhost = run['%s_distrib_hostfile' % mode] else: fhost = None if fhost: hostinfo = parse_config(fhost) run.DBG('hostinfo : %s' % hostinfo, all=True) hostrc = ResourceManager(hostinfo) else: hostrc = PureLocalResource(run) run.DBG(hostrc) return hostrc
def Histor(run, *args): """Extract the content of some issues from REX database. """ if len(args) != 2: run.parser.error(_(u"'--%s' requires two arguments") % run.current_action) iret = 0 on_machref = run.get('rep_agla', 'local') != 'local' if not on_machref: run.Mess(_(u'Only available on the AGLA machine'), '<F>_AGLA_ERROR') # 0. check imports if not imports_succeed: run.Mess(_(u'Imports of REX interface failed.'), '<F>_IMPORT_ERROR') # 1. copy input file jn = run['num_job'] ffich = get_tmpname(run, run['tmp_user'], basename='hist_input') kret = run.Copy(ffich, args[0], niverr='<F>_COPYFILE') # 2. read input file hist_content = open(ffich, 'r').read() expr = re.compile('([0-9]+)', re.MULTILINE) l_nf = [int(n) for n in expr.findall(hist_content)] # 3. open database connection try: c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug']) except Exception, msg: run.Mess(convert(msg), '<F>_DB_ERROR')
def set_rep_trav(self, reptrav, basename=''): """Set temporary directory for Code_Aster executions.""" run = magic.run if reptrav: self.local_reptrav = self.global_reptrav = reptrav if self.really(): self.global_reptrav = osp.join(reptrav, 'global') if not osp.exists(self.global_reptrav): os.makedirs(self.global_reptrav) else: self.local_reptrav = self.global_reptrav = get_tmpname(run, basename=basename) # used shared directory only if more than one node if self.really(): # and self.nbnode() > 1: shared_tmp is necessary if submission # node is not in mpihosts list # MUST not contain local_reptrav dirn = get_tmpname(run, run['shared_tmp'], basename) self.global_reptrav = osp.join(dirn, 'global') run.DBG("set_rep_trav(%s) > local=%s, global=%s" \ % (reptrav, self.local_reptrav, self.global_reptrav), stack_id=1) return self.global_reptrav
def ProxyToServer(run, *args): """Work as a proxy to a server to run an action. An export file is required to get the informations to connect the server. If the action has not a such argument, it will be the first for calling through the proxy. The other arguments are those of the action. This option is intended to be called on a client machine (directly by the gui for example). """ # The options must be passed explictly for each action because their # meaning are not necessarly the same on client and server sides. # Example : "num_job" of client has no sense on the server. # An options list can be added to ACTIONS definitions. magic.log.info('-' * 70) run.DBG("'--proxy' used for action '%s' and args : %s" % (run.current_action, args)) dact = ACTIONS.get(run.current_action) # check argument if dact is None: run.parser.error(_(u"these action can not be called through the proxy : '--%s'") \ % run.current_action) if not (dact['min_args'] <= len(args) <= dact['max_args']): run.parser.error(_(u"'--%s' : wrong number of arguments (min=%d, max=%d)") \ % (run.current_action, dact['min_args'], dact['max_args'])) # read export from arguments prof = None if dact['export_position'] < len(args): profname = args[dact['export_position']] fprof = run.PathOnly(profname) if fprof != profname: run.DBG( "WARNING: --proxy should be called on a local export file, not %s" % profname) fprof = get_tmpname(run, run['tmp_user'], basename='profil_astk') iret = run.Copy(fprof, profname, niverr='<F>_PROFILE_COPY') run.ToDelete(fprof) if fprof == "None": # the client knows that the schema does not need an export file fprof = None elif not osp.isfile(fprof): run.Mess(ufmt(_(u'file not found : %s'), fprof), '<F>_FILE_NOT_FOUND') prof = AsterProfil(fprof, run) if fprof is not None: run.DBG("Input export : %s" % fprof, prof) iret = call_plugin(run.current_action, prof, *args) if type(iret) in (list, tuple): iret = iret[0] run.Sortie(iret)
def Creer(run, *args): """Create a new issue in REX database. """ # backward compatibility: 2nd argument was optional in astk <= 1.8.3 if len(args) < 1 or len(args) > 2: run.parser.error(_(u"'--%s' requires one or two arguments") % run.current_action) iret = 0 on_machref = run.get('rep_agla', 'local') != 'local' if not on_machref: run.Mess(_(u'Only available on the AGLA machine'), '<F>_AGLA_ERROR') # 0. check imports if not imports_succeed: run.Mess(_(u'Imports of REX interface failed.'), '<F>_IMPORT_ERROR') # 1. copy issue file jn = run['num_job'] ffich = get_tmpname(run, run['tmp_user'], basename='rex_fich') kret = run.Copy(ffich, args[0], niverr='<F>_COPYFILE') # 1b. parse issue file content = open(ffich, 'r').read() d = parse_issue_file(content) if run['debug']: print3('Dict issue content : ', repr(d)) fichetude = int(d.get('FICASS') is not None) if fichetude == 1: assert len(args) > 1, "inconsistent data" # check backward compatibility fprof = get_tmpname(run, run['tmp_user'], basename='rex_prof') kret = run.Copy(fprof, args[1], niverr='<F>_PROFILE_COPY') # 3. open database connection try: c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug']) except Exception, msg: run.Mess(msg, '<F>_DB_ERROR')
def change_profile(self): """Prepare profile object. """ # initialize the profile ptest = init_profil_from(self.run, self.prof.copy()) fname = get_tmpname(self.run, self.run['tmp_user'], basename=self.testcase + '.export', pid=self.pid) ptest.set_filename(fname) self.run.DBG('profile filename set to : ', fname) del ptest['follow_output'] del ptest['rep_trav'] del ptest['detr_rep_trav'] del ptest['depart'] del ptest['mem_aster'] # update with the export to run the test lunig = ptest.get_type('unig') d_unig = None if lunig: unigest = lunig[0].path conf = build_config_from_export(self.run, ptest) build = AsterBuild(self.run, conf) d_unig = build.GetUnigest(unigest) prt = build_test_export(self.run, self.param['conf'], self.param['REPREF'], self.param['reptest'], self.testcase, self.param['resutest'], with_default=False, d_unig=d_unig) prt['nomjob'] = self.testcase ptest.update(prt) # apply facmtps try: ptest['tpsjob'] = int(float( ptest['tpsjob'][0])) * self.param['facmtps'] except Exception: pass try: ptest.args['tpmax'] = int(float( ptest.args['tpmax'])) * self.param['facmtps'] except Exception: pass ptest.update_content() return ptest
def modify(self): """Modifications for extern tool.""" if self.on_client_side: return # get executable dbg = self.prof_orig['debug'][0] if dbg == '': dbg = 'nodebug' if self.prof_orig.Get('D', typ='exec'): d_exe = self.prof_orig.Get('D', typ='exec')[0] else: d_exe = { 'path': '?', 'type': 'exec', 'isrep': False, 'compr': False, 'ul': 0 } conf = build_config_from_export(self.run, self.prof_orig) if dbg == 'nodebug': d_exe['path'] = conf.get_with_absolute_path('BIN_NODBG')[0] else: d_exe['path'] = conf.get_with_absolute_path('BIN_DBG')[0] # add valgrind command if self.prof_orig['exectool'][0] == '': self.run.Mess(_(u'"exectool" is not defined !'), '<F>_PROGRAM_ERROR') cmd = self.prof_orig['exectool'][0] if self.run.get(cmd): cmd = self.run[cmd] + ' ' else: cmd += ' ' if self.run.IsRemote(d_exe['path']): self.run.Mess( _(u'"exectool" can not be used with a remote executable.'), "<F>_ERROR") cmd += d_exe['path'] + ' "$@" 2>&1' # write script exetmp = get_tmpname(self.run, self.run['shared_tmp'], basename='front_exec') cmd += '\n' + 'rm -f %s' % exetmp open(exetmp, 'w').write(cmd) os.chmod(exetmp, 0755) d_exe['path'] = exetmp # change profile del self.new_prof['exectool'] self.new_prof.Del('D', typ='exec') self.new_prof.Set('D', d_exe)
def __init__(self, run, prof): """Initialization run : AsterRun object, prof : AsterProfil object. """ self.run = run self.prof = prof self.script = None self.change_input_script = False self.jn = os.getpid() self.btc_file = None if prof: self.btc_file = get_tmpname(self.run, self.run['tmp_user'], basename='btc_%s' % prof['nomjob'][0])
def build_config_from_export(run, prof): """Build an AsterConfig object from the default config file or the one referenced in the profile.""" from asrun.common_func import get_tmpname version_path = prof.get_version_path() lconf = prof.Get('D', typ='conf') if not lconf: ficconf = os.path.join(version_path, 'config.txt') else: ficconf = lconf[0]['path'] if run.IsRemote(ficconf): ficconf = get_tmpname(run, run['tmp_user'], basename='config.txt') run.ToDelete(ficconf) kret = run.Copy(ficconf, lconf[0]['path']) else: ficconf = run.PathOnly(ficconf) return AsterConfig(ficconf, run, version_path)
def ConvertToHtml(run, *args): """Convert a file into html format. """ run.PrintExitCode = False # ----- check argument if len(args) != 1: run.parser.error( _(u"'--%s' requires one argument") % run.current_action) if run.get('output') is None: run.parser.error( _(u"'--%s' requires --output=FILE option") % run.current_action) ftmp = get_tmpname(run, run['tmp_user'], basename='convert_html') run.ToDelete(ftmp) kret = run.Copy(ftmp, args[0], niverr='<F>_COPYFILE') out = convert2html(ftmp) out.sortieHTML(run['output'])
def SendMail(run, *args): """Send the content of a file by mail. """ # check argument if len(args) != 1: run.parser.error( _(u"'--%s' requires one argument") % run.current_action) # content file fcontent = get_tmpname(run, run['tmp_user'], basename='mail_content') run.ToDelete(fcontent) kret = run.Copy(fcontent, args[0], niverr='<F>_PROFILE_COPY') if run['report_to'] == '': run.Mess(_(u"no email address provided !"), '<F>_ERROR') run.SendMail(dest=run['report_to'], text=open(fcontent, 'r').read(), subject='From as_run/SendMail')
def Tail(run, *args): """Output the last part of fort.6 file or filter lines matching a pattern. """ if len(args) < 5: run.parser.error(_(u"'--%s' takes at least %d arguments (%d given)") % \ (run.current_action, 5, len(args))) elif len(args) > 6: run.parser.error(_(u"'--%s' takes at most %d arguments (%d given)") % \ (run.current_action, 6, len(args))) # arguments njob, nomjob, mode, fdest, nbline = args[:5] expression = None if len(args) > 5: expression = args[5] # allow to customize of the executed function worker = Func_tail if run.get('schema_tail_exec'): worker = get_plugin(run['schema_tail_exec']) run.DBG("calling plugin : %s" % run['schema_tail_exec']) etat, diag, s_out = worker(run, njob, nomjob, mode, nbline, expression) print_tail_result(nomjob, njob, etat, diag) if s_out == "": s_out = _(u'the output is empty (job ended ?)') # exit if job isn't running run.PrintExitCode = False if run.get('result_to_output') or fdest == 'None': run.DBG(_(u'tail put to output')) print3(s_out) else: # send output file if run.IsRemote(fdest): ftmp = get_tmpname(run, run['tmp_user'], basename='tail') open(ftmp, 'w').write(convert(s_out)) jret = run.Copy(fdest, ftmp) else: fdest = run.PathOnly(fdest) open(fdest, 'w').write(convert(s_out)) run.DBG(ufmt(u'output written to : %s', fdest))
def __init__(self, run, filename=None, prof=None, pid=None, differ_init=False): """Initializations """ BaseCalcul.__init__(self) assert filename or prof, 'none of (filename, prof) provided!' self.run = run if pid is None: self.pid = self.run['num_job'] else: self.pid = pid self.studyid = self.pid if prof is not None: self.prof = prof else: # ----- profile filename fprof = get_tmpname(self.run, self.run['tmp_user'], basename='profil_astk') self.run.ToDelete(fprof) kret = self.run.Copy(fprof, filename, niverr='<F>_PROFILE_COPY') self.prof = AsterProfil(fprof, self.run) if self.prof['nomjob'][0] == '': self.prof['nomjob'] = 'unnamed' # attributes self.dict_info = None self.as_exec_ref = self.run.get('as_exec_ref') self.diag = '?' self.__initialized = False if not differ_init: self.finalize_init()
def Parametric(run, prof, runner, numthread='auto', **kargs): """Run a parametric study. """ run.print_timer = True # 1. ----- initializations jn = run['num_job'] # 1.2. rep_trav from profile or from run[...] reptrav = runner.reptrav() run.Mess(_(u'Code_Aster parametric execution'), 'TITLE') runner.set_cpuinfo(1, 1) # ----- how many threads ? try: numthread = int(numthread) except (TypeError, ValueError): numthread = run.GetCpuInfo('numthread') # 1.3. content of the profile if not prof.Get('D', typ='distr'): run.Mess(_(u'"distr" file is necessary'), '<F>_FILE_NOT_FOUND') else: fdist = prof.Get('D', typ='distr')[0]['path'] if run.IsRemote(fdist): tmpdist = get_tmpname(run, run['tmp_user'], basename='distr') run.ToDelete(tmpdist) kret = run.Copy(tmpdist, fdist) fdist = tmpdist else: fdist = run.PathOnly(fdist) dist_cnt = open(fdist, 'r').read() try: keywords = get_distribution_data(cleanCR(dist_cnt)) except Exception, msg: run.Mess( _(u"Error in the distr file: {0}").format(str(msg)), '<F>_ERROR')
def modify(self): """Modifications for meshtool service. """ super(ProfileModifierMeshtool, self).modify() # job name self.new_prof['nomjob'] = '%s_mesh' % self.prof_orig['nomjob'][0] # commands file fcomm = os.path.join(datadir, 'meshtool.comm') self.new_prof.Set('D', { 'path': fcomm, 'ul': 1, 'type': 'comm', 'isrep': False, 'compr': False }) # special : mesh IN (unit 71), mesh OUT (unit 72) assert len(self.special) >= 3 self.new_prof.parse("""%s 71\n%s 72""" % tuple(self.special[1:3])) # parameter file : field 4 and next fpara = get_tmpname(self.run, self.run['tmp_user'], basename='meshtool.para') self.run.Delete(fpara) txt = os.linesep.join(self.special[3:]) open(fpara, 'w').write(txt) self.new_prof.Set( 'D', { 'path': self.check_filename(fpara), 'ul': 70, 'type': 'libr', 'isrep': False, 'compr': False })
def copyfileR(run, df, copybase=True, alarm=True): """Copy results from current directory into destination given by `df`. Aster bases are copied only if copybase is True. Raise only <E> if an error occurs run CheckOK() after. """ # list of files lf = [] isdir = False # 1. ----- files if df['ul'] != 0 or df['type'] in ('nom', ): # if logical unit is set : fort.* if df['ul'] != 0: lf.append('fort.%d' % df['ul']) elif df['type'] == 'nom': lf.append(osp.basename(df['path'])) # 2. ----- bases and directories (ul=0) else: isdir = True # base if df['type'] == 'base' and copybase: lf.extend(glob('glob.*')) lf.extend(glob('pick.*')) # bhdf elif df['type'] == 'bhdf' and copybase: lbas = glob('bhdf.*') if len(lbas) == 0: if alarm: run.Mess(_(u"No 'bhdf' found, saving 'glob' instead"), '<A>_COPY_BASE') lbas = glob('glob.*') lf.extend(lbas) lf.extend(glob('pick.*')) # repe elif df['type'] == 'repe': rep = 'REPE_OUT' lfrep = glob(osp.join(rep, '*')) if len(lfrep) == 0 and alarm: run.Mess(ufmt(_(u"%s directory is empty !"), rep), '<A>_COPY_REPE') lf.extend(lfrep) # 3. ----- compression kret = 0 if df['compr']: lfnam = lf[:] lf = [] for fnam in lfnam: kret, f = run.Gzip(fnam, niverr='<E>_COMPRES_ERROR', verbose=True) if kret == 0: lf.append(f) else: lf.append(fnam) run.Mess(_(u"Warning: The uncompressed file will be returned " u"without changing the target filename\n(eventually " u"ending with '.gz' even if it is not compressed; " u"you may have to rename it before use)."), '<A>_COPYFILE') # 4. ----- copy if len(lf) > 0: # create destination if isdir: kret = run.MkDir(df['path'], '<E>_MKDIR_ERROR') else: if len(lf) > 1: run.Mess(ufmt(_(u"""Only the first one of [%s] is copied."""), ', '.join(lf)), '<A>_COPYFILE') lf = [lf[0],] kret = run.MkDir(osp.dirname(df['path']), '<E>_MKDIR_ERROR') # copy if kret == 0: lfc = lf[:] for fname in lfc: if not osp.exists(fname): if alarm: run.Mess(ufmt(u"no such file or directory: %s", fname), '<A>_COPYFILE') lf.remove(fname) if len(lf) > 0: kret = run.Copy(df['path'], niverr='<E>_COPY_ERROR', verbose=True, *lf) # save base if failure if kret != 0: rescue = get_tmpname(run, basename='save_results') run.Mess(ufmt(_(u"Saving results in a temporary directory (%s)."), rescue), '<A>_COPY_RESULTS', store=True) kret = run.MkDir(rescue, chmod=0700) kret = run.Copy(rescue, niverr='<E>_COPY_ERROR', verbose=True, *lf)
def Func_actu(run, *args): """Return state, diagnosis, node, cpu time and working directory of a job. """ if len(args) != 3: run.parser.error(_(u"'--%s' takes exactly %d arguments (%d given)") % \ (run.current_action, 3, len(args))) njob, nomjob, mode = args # defaults etat = '_' diag = '_' node = '_' tcpu = '_' wrk = '_' queue = '_' psout = '' # the real job id may differ jobid = str(njob) # astk profile pr_astk = osp.join(run['flasheur'], '%s.p%s' % (nomjob, njob)) prof = None if osp.isfile(pr_astk): prof = AsterProfil(pr_astk, run) wrk = prof['rep_trav'][0] or '_' # 1. get information about the job # 1.1. batch mode if mode == "batch": m = 'b' scheduler = BatchSystemFactory(run, prof) etat, diag, node, tcpu, wrkb, queue = scheduler.get_jobstate( njob, nomjob) # 1.2. interactive mode elif mode == "interactif": m = 'i' # if it doesn't exist the job is ended etat = "ENDED" if prof is not None: node = prof['noeud'][0] else: run.Mess(_(u'unexpected mode : %s') % mode, '<F>_UNEXPECTED_VALUE') # 2. query the process if node != '_': if mode == "interactif" or tcpu == '_': jret, psout = run.Shell(run['ps_cpu'], mach=node) # ended ? if mode == "interactif" and psout.find('btc.%s' % njob) > -1: etat = "RUN" # 3.1. the job is ended if etat == "ENDED": fdiag = osp.join(run['flasheur'], '%s.%s%s' % (nomjob, m, njob)) if osp.isfile(fdiag): diag = open(fdiag, 'r').read().split(os.linesep)[0] or "?" if diag == '?': diag = '<F>_SYSTEM' # try to find something in output fout = osp.join(run['flasheur'], '%s.o%s' % (nomjob, njob)) if osp.isfile(fout): f = open(fout, 'r') for line in f: if line.find('--- DIAGNOSTIC JOB :') > -1: diag = line.split()[4] elif line.find('Cputime limit exceeded') > -1: diag = '<F>_CPU_LIMIT_SYSTEM' f.close() # copy fort.6 to '.o' if node != '_': ftcp = get_tmpname(run, run['tmp_user'], basename='actu') # same name as in the btc script generated by calcul.py wrk6 = get_nodepara(node, 'rep_trav', run['rep_trav']) fort6 = osp.join(wrk6, '%s.%s.fort.6.%s' % (nomjob, njob, m)) jret = run.Copy(ftcp, '%s:%s' % (node, fort6), niverr='SILENT') if osp.isfile(ftcp): txt = [os.linesep * 2] txt.append('=' * 48) txt.append( '===== Pas de diagnostic, recopie du fort.6 =====') txt.append('=' * 48) txt.append(open(ftcp, 'r').read()) txt.append('=' * 48) txt.append('=' * 48) txt.append(os.linesep * 2) f = open(fout, 'a') f.write(os.linesep.join(txt)) f.close() else: # 3.2. job is running if etat in ('RUN', 'SUSPENDED'): # working directory if wrk == '_': wrk = get_tmpname(run, basename=mode, node=node, pid=njob) if etat == 'RUN' and tcpu == '_': # tcpu may have been retrieved upper l_tcpu = [] for line in psout.split(os.linesep): if re.search('\-\-num_job=%s' % njob, line) != None and \ re.search('\-\-mode=%s' % mode, line) != None: l_tcpu.append( re.sub('\..*$', '', line.split()[0]).replace('-', ':')) if len(l_tcpu) > 0: try: tcpu = dhms2s(l_tcpu) except ValueError: pass # 4. return the result if node == "": node = "_" return etat, diag, node, tcpu, wrk, queue
def RunAster(run, *args): """Allow to run Code_Aster with or without compiling additional source files, check a development, run a list of test cases... """ run.print_timer = True prev = os.getcwdu() # ----- check argument if len(args) != 1: run.parser.error( _(u"'--%s' requires one argument") % run.current_action) # 1. ----- initializations jn = run['num_job'] fprof = get_tmpname(run, run['tmp_user'], basename='profil_astk') run.ToDelete(fprof) # 1.1. ----- check argument type # 1.1.1. ----- use profile from args if isinstance(args[0], AsterProfil): prof = args[0].copy() prof.WriteExportTo(fprof) forig = fprof else: # 1.1.2. ----- read profile from args forig = args[0] kret = run.Copy(fprof, forig, niverr='<F>_PROFILE_COPY') prof = AsterProfil(fprof, run) if not run.IsRemote(forig): export_fname = run.PathOnly(get_absolute_path(forig)) prof.absolutize_filename(export_fname) if not prof['mode'][0] in ('batch', 'interactif'): run.Mess(_(u"Unknown mode (%s), use 'interactif' instead") % \ repr(prof['mode'][0]), 'UNEXPECTED_VALUE') prof['mode'] = ['interactif'] run.DBG("Input export : %s" % fprof, prof) # 1.2. get AsterConfig and AsterBuild objects REPREF = prof.get_version_path() conf = build_config_from_export(run, prof) build = AsterBuild(run, conf) DbgPara = { 'debug': { 'exe': conf['BIN_DBG'][0], 'suff': conf['BINOBJ_DBG'][0], 'libast': conf['BINLIB_DBG'][0], 'libfer': conf['BINLIBF_DBG'][0] }, 'nodebug': { 'exe': conf['BIN_NODBG'][0], 'suff': conf['BINOBJ_NODBG'][0], 'libast': conf['BINLIB_NODBG'][0], 'libfer': conf['BINLIBF_NODBG'][0] }, } # 1.3. set environment depending on version for f in conf.get_with_absolute_path('ENV_SH'): run.AddToEnv(f) # 1.4. set runner parameters klass = Runner # allow to customize of the execution objects if run.get('schema_execute'): schem = get_plugin(run['schema_execute']) run.DBG("calling plugin : %s" % run['schema_execute']) klass = schem(prof) runner = klass(conf.get_defines()) iret = runner.set_cpuinfo(prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0]) if iret == 1: run.Mess( ufmt( _(u"%s is not a MPI version of Code_Aster. " "The number of nodes/processors must be equal to 1."), REPREF), "<F>_INVALID_PARAMETER") elif iret != 0: run.Mess(_(u"incorrect value for mpi_nbnoeud (%s) or mpi_nbcpu (%s)") \ % (prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0]), '<F>_INVALID_PARAMETER') # 1.5. rep_trav from profile or from run[...] reptrav = runner.set_rep_trav(prof['rep_trav'][0], prof['mode'][0]) # write reptrav in the export prof['rep_trav'] = reptrav prof.WriteExportTo(prof.get_filename()) #XXX overrides the original export if forig != prof.get_filename(): run.Copy(forig, prof.get_filename(), niverr='<A>_ALARM') # add reptrav to LD_LIBRARY_PATH (to find dynamic libs provided by user) old = os.environ.get("LD_LIBRARY_PATH", "") os.environ["LD_LIBRARY_PATH"] = (reptrav + os.pathsep + old).strip( os.pathsep) # do not reinitialize rep_trav if if prof['prep_env'][0] not in NO_VALUES: run.MkDir(reptrav, chmod=0700) if prof['detr_rep_trav'][0] not in NO_VALUES: run.ToDelete(reptrav) # 1.6. copy profile in rep_trav kret = run.Copy(osp.join(reptrav, jn + '.export'), fprof) # ... and config file as ./config.txt conf.WriteConfigTo(osp.join(reptrav, 'config.txt')) # 1.7. debug/nodebug dbg = prof['debug'][0] if dbg == '': dbg = 'nodebug' # 1.8. default values exetmp = osp.join(REPREF, DbgPara[dbg]['exe']) cmdetmp = osp.join(REPREF, conf['BINCMDE'][0]) eletmp = osp.join(REPREF, conf['BINELE'][0]) # 2. ----- read profile values # it's valid because exec, cmde and ele must appear only once # these values will be overidden if they are available in reptrav # after an occurence of 'make_...' if prof.Get('DR', 'exec'): exetmp = prof.Get('DR', 'exec')[0]['path'] if prof.Get('DR', 'cmde'): cmdetmp = prof.Get('DR', 'cmde')[0]['path'] if prof.Get('DR', 'ele'): eletmp = prof.Get('DR', 'ele')[0]['path'] # order of actions : list_actions = [ 'make_exec', 'make_cmde', 'make_ele', 'make_etude', 'make_dbg', 'make_env', 'astout', 'distribution', 'multiple', 'exec_crs', 'exec_crp' ] # 3. ==> Let's go ! # 3.0. check if we know what to do for act in prof['actions']: if act == '': run.Mess(_(u'nothing to do'), 'OK') elif not act in list_actions: run.Mess(_(u'unknown action : %s') % act, '<A>_ALARM') # check if the version allows developments if conf['DEVEL'][0] in NO_VALUES and \ ( 'make_exec' in prof['actions'] or \ 'make_cmde' in prof['actions'] or \ 'make_ele' in prof['actions'] ): run.Mess( _(u'The configuration of this version does not allow ' 'user developments.'), '<F>_ERROR') # # 3.1. ----- make_exec # iret = 0 if 'make_exec' in prof['actions']: run.DBG(u'Start make_exec action') exetmp = osp.join(reptrav, 'aster.exe') tit = _(u'Compilation of source files') run.Mess(tit, 'TITLE') run.timer.Start(tit) repact = osp.join(reptrav, 'make_exec') repobj = osp.join(repact, 'repobj') run.MkDir(repact) lf = [] for typ in ('c', 'f', 'f90'): for rep in [l['path'] for l in prof.Get('D', typ=typ)]: jret, lbi = build.Compil(typ.upper(), rep, repobj, dbg, rep_trav=repact, error_if_empty=True, numthread='auto') iret = max(iret, jret) lf.extend(lbi) # liste des fichiers surchargés vers = get_aster_version(REPREF) vers = '.'.join(vers[:3]) fsurch = osp.join(repact, 'surchg.f') listsurcharge(vers, fsurch, lf) jret, lbi = build.Compil('F', fsurch, repobj, dbg, repact) run.timer.Stop(tit) run.CheckOK() tit = _(u'Build executable') run.Mess(tit, 'TITLE') run.timer.Start(tit) libaster = osp.join(REPREF, DbgPara[dbg]['libast']) libferm = osp.join(REPREF, DbgPara[dbg]['libfer']) # for backward compatibility if run.IsDir(libaster): libaster = osp.join(libaster, 'lib_aster.lib') if run.IsDir(libferm): libferm = osp.join(libferm, 'ferm.lib') lobj = glob(osp.join(repobj, '*.o')) # build an archive if there are more than NNN object files if len(lobj) > 500: run.timer.Stop(tit) tit2 = _(u'Add object files to library') run.timer.Start(tit2) libtmp = osp.join(repobj, 'libsurch.a') run.Copy(libtmp, libaster) kret = build.Archive(repobj, libtmp, force=True) lobj = [] libaster = libtmp run.timer.Stop(tit2) run.timer.Start(tit) kret = build.Link(exetmp, lobj, libaster, libferm, repact) run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit, num=999) if prof.Get('R', typ='exec'): exe = prof.Get('R', typ='exec')[0] run.Delete(exe['path'], remove_dirs=False) iret = run.MkDir(osp.dirname(exe['path'])) iret = run.Copy(exe['path'], exetmp) exedata = prof.Get('D', typ='exec') if exedata and exedata[0]['path'] != exe['path']: exetmp = exedata[0]['path'] run.timer.Stop(tit) run.Mess(_(u'Code_Aster executable successfully created'), 'OK') # # 3.2. ----- make_cmde # if 'make_cmde' in prof['actions']: run.DBG(u'Start make_cmde action') tit = _(u"Compilation of commands catalogue") cmdetmp = osp.join(reptrav, 'cata_commande') run.timer.Start(tit) repact = osp.join(reptrav, 'make_cmde') run.MkDir(repact) kargs = { 'exe': exetmp, 'cmde': cmdetmp, } kargs['capy'] = [l['path'] for l in prof.Get('D', typ='capy')] lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] jret = build.CompilCapy(REPREF, repact, **kargs) #i18n=True, run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit) if prof.Get('R', typ='cmde'): cmde = prof.Get('R', typ='cmde')[0] iret = run.MkDir(cmde['path']) iret = run.Copy(cmde['path'], osp.join(cmdetmp, 'cata*.py*')) run.timer.Stop(tit) # # 3.3. ----- make_ele # if 'make_ele' in prof['actions']: run.DBG(u'Start make_ele action') tit = _(u"Compilation of elements") eletmp = osp.join(reptrav, 'elem.1') run.timer.Start(tit) repact = osp.join(reptrav, 'make_ele') run.MkDir(repact) kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, } kargs['cata'] = [l['path'] for l in prof.Get('D', typ='cata')] lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] jret = build.CompilEle(REPREF, repact, **kargs) run.timer.Stop(tit) run.CheckOK() tit = _(u'Copying results') run.timer.Start(tit) if prof.Get('R', typ='ele'): ele = prof.Get('R', typ='ele')[0] iret = run.MkDir(osp.dirname(ele['path'])) iret = run.Copy(ele['path'], eletmp) run.timer.Stop(tit) # # 3.4. ----- make_env / make_etude / make_dbg # if 'make_env' in prof['actions'] or 'make_etude' in prof['actions'] or \ 'make_dbg' in prof['actions']: run.DBG(u'Start make_etude/make_env/make_dbg action') os.chdir(reptrav) run.Mess(_(u'Code_Aster execution'), 'TITLE') # 3.4.1. prepare reptrav to run Code_Aster (proc# = 0) only_env = 'make_env' in prof['actions'] kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'lang': prof['lang'][0], 'only_env': only_env, } lfun = prof.Get('D', typ='unig') if lfun: kargs['unigest'] = build.GetUnigest(lfun[0]['path']) if prof.Get('D', typ='py'): kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')] tit = _(u'Preparation of environment') run.timer.Start(tit) run.Mess(ufmt(_(u'prepare environment in %s'), reptrav)) if prof['prep_env'][0] != 'no': build.PrepEnv(REPREF, reptrav, dbg=dbg, **kargs) else: run.Mess(_(u'... skipped (%s = no) !') % 'prep_env', 'SILENT') run.timer.Stop(tit) # 3.4.2. copy datas (raise <E> errors if failed) tit = _(u'Copying datas') run.Mess(tit, 'TITLE') run.timer.Start(tit) if prof['copy_data'][0] not in NO_VALUES: copyfiles(run, 'DATA', prof) else: run.Mess(_(u'... skipped (%s = no) !') % 'copy_data', 'SILENT') print3(os.getcwdu()) run.timer.Stop(tit) # 3.4.3. execution diag, tcpu, tsys, ttot, copybase = execute(reptrav, multiple=False, with_dbg='make_dbg' in prof['actions'], only_env=only_env, runner=runner, run=run, conf=conf, prof=prof, build=build, exe=exetmp) if not 'make_env' in prof['actions']: # 3.4.4. copy results tit = _(u'Copying results') run.Mess(tit, 'TITLE') run.timer.Start(tit) if prof['copy_result'][0] not in NO_VALUES: emit_alarm = prof['copy_result_alarm'][0] not in NO_VALUES copyfiles(run, 'RESU', prof, copybase, emit_alarm) else: run.Mess( _(u'... skipped (%s = no) !') % 'copy_result', 'SILENT') run.timer.Stop(tit) run.Mess(_(u'Code_Aster run ended'), diag) # 3.4.5. add .resu/.erre to output for testcases ctest = prof['parent'][0] == "astout" if ctest: run.Mess(_(u'Content of RESU file'), 'TITLE') run.FileCat('fort.8', magic.get_stdout()) run.Mess(_(u'Content of ERROR file'), 'TITLE') run.FileCat('fort.9', magic.get_stdout()) # 3.4.6. notify the user if prof['notify'][0]: content = _( '[Code_Aster] job %(job)s on %(server)s ended: %(diag)s') content = content % { 'job': prof.get_jobname(), 'diag': diag, 'server': prof['serveur'][0], } dest = ','.join(prof['notify']) run.SendMail(dest=dest, text=content, subject=content.splitlines()[0]) run.Mess(_(u'Email notification sent to %s') % dest) run.CheckOK() os.chdir(prev) # 3.5. ----- astout if 'astout' in prof['actions']: run.DBG(u'Start astout action') kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'numthread': prof['numthread'][0], } os.chdir(reptrav) RunAstout(run, conf, prof, runner=runner, **kargs) os.chdir(prev) # 3.6. ----- distribution if 'distribution' in prof['actions']: run.DBG(u'Start distribution action') kargs = { 'exe': exetmp, 'cmde': cmdetmp, 'ele': eletmp, 'numthread': prof['numthread'][0], } Parametric(run, prof, runner=runner, **kargs) # 3.7. ----- multiple if 'multiple' in prof['actions']: run.DBG(u'Start multiple action') Multiple(run, prof, runner=runner, numthread=prof['numthread'][0]) # 4. ----- clean up if 'make_env' in prof['actions'] and prof['detr_rep_trav'][ 0] not in NO_VALUES: run.DoNotDelete(reptrav)
def StartNow(run, *args): """Start quickly a simple execution using files in arguments. """ if not run["silent"]: run.Mess( _(u"This functionnality is still in a beta state of development " u"and may be removed a future release, or may never be improved. " u"Please use --silent option to ignore this message the next time." )) # ----- check argument if not run.get('aster_vers'): run.parser.error( _(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option." )) if len(args) < 1: run.parser.error( _(u"'--%s' requires at least one argument") % run.current_action) # build export lf = [osp.abspath(f) for f in args] prof = build_export_from_files(run, lf, with_results=True) prof = use_options(run, prof) # development files surch_pyt = run.get('surch_pyt', []) if surch_pyt: for obj in surch_pyt.split(','): prof.Set( 'D', { 'path': osp.abspath(obj), 'type': 'py', 'ul': 0, 'isrep': osp.isdir(osp.abspath(obj)), 'compr': False, }) exetmp = None surch_fort = run.get('surch_fort', []) if surch_fort: for obj in surch_fort.split(','): prof.Set( 'D', { 'path': osp.abspath(obj), 'type': 'f', 'ul': 0, 'isrep': osp.isdir(osp.abspath(obj)), 'compr': False, }) exetmp = get_tmpname(run, run['tmp_user'], basename='executable') prof.Set( 'DR', { 'path': exetmp, 'type': 'exec', 'ul': 0, 'isrep': False, 'compr': False, }) if exetmp is not None: prof['actions'] = prof['actions'] + ['make_exec'] # try to grab all results files if run["copy_all_results"]: resudir = os.getcwdu() lcomm = prof.get_type('comm') if len(lcomm) > 0: jobname = osp.splitext(osp.basename(lcomm[0].path))[0] else: jobname = 'unamed' add_all_results(prof, resudir, jobname) run.Mess(_(u"Profile used :"), 'TITLE') run.Mess(os.linesep + prof.get_content(), 'SILENT') # execution RunAster(run, prof)
def RunAstout(run, conf, prof, runner, numthread='auto', **kargs): """Run a list of test cases... """ run.print_timer = True # 1. ----- initializations REPREF = prof.get_version_path() # 1.2. rep_trav from profile or from run[...] reptrav = runner.reptrav() run.Mess(_(u'Code_Aster tests execution'), 'TITLE') runner.set_cpuinfo(1, 1) # ----- how many threads ? try: numthread = int(numthread) except (TypeError, ValueError): numthread = run.GetCpuInfo('numthread') # 1.3. content of the profile ltest = osp.join(reptrav, 'tests.list') if not prof.Get('D', typ='list'): run.Mess(_(u'no list of tests found'), '<E>_NO_TEST_LIST') else: for dli in prof.Get('D', typ='list'): if run.IsRemote(dli['path']): tmplist = get_tmpname(run, run['tmp_user'], basename='list') run.ToDelete(tmplist) kret = run.Copy(tmplist, dli['path']) run.FileCat(tmplist, ltest) else: tmplist = run.PathOnly(dli['path']) run.FileCat(tmplist, ltest) if not prof.Get('D', typ='rep_test'): reptest = [] else: reptest = [r['path'] for r in prof.Get('D', typ='rep_test')] if not prof.Get('R', typ='resu_test'): run.Mess(_(u'no result directory found'), '<E>_NO_RESU_DIR') else: resutest = prof.Get('R', typ='resu_test')[0]['path'] if run.IsRemote(resutest): run.Mess(_(u'the result directory must not be on a remote host'), '<F>_ERROR') resutest = run.PathOnly(resutest) #peter.zhang, for cygwin resutest = resutest.replace('\\', '/') resutest = resutest.replace('/cygdrive/', '') if resutest.find(':/') < 0: resutest = resutest.replace('/', ':/', 1) run.MkDir(resutest) run.Delete(osp.join(resutest, 'RESULTAT')) run.Delete(osp.join(resutest, 'NOOK')) flashdir = osp.join(resutest, 'flash') prfl = prof.Get('R', typ='flash') if prfl: if prfl[0]['path'] == "None": flashdir = None else: flashdir = prfl[0]['path'] facmtps = 1. nbmaxnook = 5 cpresok = 'RESNOOK' try: if prof['facmtps'][0] != '': facmtps = float(prof['facmtps'][0]) except ValueError: run.Mess( _(u'incorrect value for %s : %s') % ('facmtps', prof['facmtps'][0])) try: if prof['nbmaxnook'][0] != '': nbmaxnook = int(prof['nbmaxnook'][0]) except ValueError: run.Mess( _(u'incorrect value for %s : %s') % ('nbmaxnook', prof['nbmaxnook'][0])) if prof['cpresok'][0] != '': cpresok = prof['cpresok'][0] run.CheckOK() # get the list of the tests to execute iret, list_tests = get_list(ltest, unique=True) if iret != 0: run.Mess(_(u'error during reading file : %s') % ltest, '<F>_ERROR') nbtest = len(list_tests) # should we run only nook tests ? if run['only_nook'] or prof['only_nook'][0] in YES_VALUES: l_dirs = reptest + [resutest] old_run_result = getDiagnostic(run, l_dirs, list_tests) run.DBG('getDiagnostic result', old_run_result, all=True) nbini = len(list_tests) list_tests = [] for test, dres in old_run_result.items(): if test.startswith('__'): continue if run.GetGrav(dres['diag']) >= run.GetGrav('NOOK'): list_tests.append(test) nbtest = len(list_tests) print3( _(u""" --- %d test-cases to run initially %d test-cases previously failed (or their results have not been found) and will be run again """) % (nbini, nbtest)) random.shuffle(list_tests) # suivi des calculs partagé entre 'numthread' threads tit = _(u"Checking hosts") hostrc = get_hostrc(run, prof) run.timer.Start(tit) n_avail, n_tot = hostrc.CheckHosts(run, numthread=numthread) run.timer.Stop(tit) run.Mess( _(u'Number of available hosts : %d/%d') % (n_avail, n_tot), "SILENT") if n_avail < 1: run.Mess(_(u"No available host. Run cancelled."), "<F>_INVALID_PARAMETER") # timeout before rejected a job = tpsjob try: timeout = prof.get_timeout() except Exception, reason: run.Mess( _(u"incorrect value for tpsjob : %s") % reason, '<F>_INVALID_PARAMETER')
def change_profile(self): """Prepare profile object. """ prof = self.prof.copy() fname = get_tmpname(self.run, self.run['tmp_user'], basename=self.label + '.export', pid=self.pid) prof.set_filename(fname) self.run.DBG('profile filename set to : ', fname) prof['actions'] = 'make_etude' prof['nomjob'] = '%s-%s' % (self.prof['nomjob'][0], self.label) # restore master parameters for i in MASTER_PARAMETERS: value = prof['MASTER_%s' % i][0] if value != "": prof[i] = value # delete unused entries assert prof.Get('R', typ='repe') del prof['follow_output'] del prof['rep_trav'] del prof['detr_rep_trav'] del prof['depart'] prof.Del('D', typ='distr') compress = prof.Get('R', typ='repe')[0]['compr'] prof.Del('R', typ='repe') # add repe_out #peter.zhang, for cygwin mydir = osp.join(self.resudir, self.label, 'REPE_OUT') mydir = mydir.replace('\\', '/') prof.Set( 'R', { 'type': 'repe', 'isrep': True, 'ul': 0, 'path': mydir, 'compr': compress, }) # add base or bhdf as data type_base, compress = prof.get_base('D') if type_base: path = prof.Get('D', typ=type_base)[0]['path'] prof.Del('D', typ=type_base) #peter.zhang, for cygwin mydir = osp.join(path, self.label, type_base) mydir = mydir.replace('\\', '/') prof.Set( 'D', { 'type': type_base, 'isrep': True, 'ul': 0, 'path': mydir, 'compr': compress, }) # add base or bhdf as result type_base, compress = prof.get_base('R') if type_base: prof.Del('R', typ=type_base) #peter.zhang, for cygwin mydir = osp.join(self.resudir, self.label, type_base) mydir = mydir.replace('\\', '/') prof.Set( 'R', { 'type': type_base, 'isrep': True, 'ul': 0, 'path': mydir, 'compr': compress, }) # change comm files lcomm = prof.Get('D', typ='comm') prof.Del('D', typ='comm') for i, df in enumerate(lcomm): fcom = osp.join(self.resudir, self.label, 'command_%d.comm' % i) #peter.zhang, for cygwin fcom = fcom.replace('\\', '/') dest = fcom if df['compr']: dest = dest + '.gz' kret = self.run.Copy(dest, df['path'], niverr='<E>_COPY_ERROR') if kret == 0 and df['compr']: kret, bid = self.run.Gunzip(dest, niverr='<E>_UNCOMPRESS') df.update({'compr': False, 'path': fcom}) prof.Set('D', df) self.change_comm_files(dest) # move all results (as files) in resudir/label for dicf in prof.resu: if dicf['isrep']: continue dicf['path'] = osp.join(self.resudir, self.label, osp.basename(dicf['path'])) #peter.zhang, for cygwin dicf['path'] = dicf['path'].replace('\\', '/') return prof
# 2. read input file hist_content = open(ffich, 'r').read() expr = re.compile('([0-9]+)', re.MULTILINE) l_nf = [int(n) for n in expr.findall(hist_content)] # 3. open database connection try: c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug']) except Exception, msg: run.Mess(convert(msg), '<F>_DB_ERROR') histor = build_histor(run, l_nf, c) # 5. copy histor file ffich = get_tmpname(run, run['tmp_user'], basename='hist_output') open(ffich, 'w').write(repr(histor)) kret = run.Copy(args[1], ffich, niverr='<F>_COPYFILE') run.Mess(_(u"Histor successfully generated."), 'OK') def Solder(run, *args): """Fill corrVdev/corrVexpl fields and close issues found in a histor file. """ if not run.get('aster_vers'): run.parser.error(_(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option.")) if len(args) != 1: run.parser.error(_(u"'--%s' requires one argument") % run.current_action) on_machref = run.get('rep_agla', 'local') != 'local' if not on_machref: